diff --git a/app/config/config.yml b/app/config/config.yml index c6f65ad..9bf88c9 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -7,8 +7,8 @@ imports: # http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration parameters: locale: en - # rebrickable csv files root URL - rebrickable_url: 'http://rebrickable.com/media/downloads/' + # rebrickable csv files root URL (http://rebrickable.com/media/downloads/ or local dir containing csv files) + rebrickable_csv_url: 'http://rebrickable.com/media/downloads/' framework: #esi: ~ diff --git a/src/AppBundle/Command/LoadRebrickableDataCommand.php b/src/AppBundle/Command/LoadRebrickableDataCommand.php index a9c441e..71d7135 100644 --- a/src/AppBundle/Command/LoadRebrickableDataCommand.php +++ b/src/AppBundle/Command/LoadRebrickableDataCommand.php @@ -12,8 +12,8 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand { $this ->setName('app:load:rebrickable') - ->setDescription('Loads Rebrickable database') - ->setHelp('This command allows you to..'); + ->setDescription('Loads Rebrickable data about sets and parts into database.') + ->setHelp('This command allows you to load Rebrickable CSV files containing information about sets and parts into database.'); } protected function execute(InputInterface $input, OutputInterface $output) diff --git a/src/AppBundle/Service/Loader/RebrickableLoader.php b/src/AppBundle/Service/Loader/RebrickableLoader.php new file mode 100644 index 0000000..7df7f91 --- /dev/null +++ b/src/AppBundle/Service/Loader/RebrickableLoader.php @@ -0,0 +1,166 @@ +rebrickable_url = $rebrickable_url; + } + + public function loadTables() + { + $connection = $this->em->getConnection(); + $connection->beginTransaction(); + + try { + $this->loadCsvFiles(); + + $connection->prepare('SET foreign_key_checks = 0;')->execute(); + $this->truncateTables(); + $connection->prepare('SET foreign_key_checks = 1;')->execute(); + + $this->output->writeln([ + 'Truncated rebrickable database tables.', + '------------------------------------------------------------------------------', + 'Loading CSV files into database...', + ]); + + $this->loadColorTable($this->csvFile['colors']); + $this->loadCategoryTable($this->csvFile['part_categories']); + $this->loadPartTable($this->csvFile['parts']); + $this->loadThemeTable($this->csvFile['themes']); + $this->loadSetTable($this->csvFile['sets']); + $this->loadInventoryTable($this->csvFile['inventories']); + $this->loadInventorySetTable($this->csvFile['inventory_sets']); + + $connection->prepare('SET foreign_key_checks = 0;')->execute(); + $this->loadInventoryPartTable($this->csvFile['inventory_parts']); + $connection->prepare('SET foreign_key_checks = 1;')->execute(); + $this->addMissingParts(); + + $connection->commit(); + + $this->output->writeln('Rebrickable database loaded successfully!'); + } catch (\Exception $e) { + $connection->rollBack(); + throw $e; + } + } + + private function loadCSVFiles() + { + $array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors']; + + $this->output->writeln([ + 'Loading Rebrickable CSV files', + '------------------------------------------------------------------------------', + ]); + + foreach ($array as $item) { + $this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv'); + } + + $this->output->writeln([ + 'Done', + '------------------------------------------------------------------------------', + ]); + } + + private function truncateTables() + { + $query = ' + TRUNCATE TABLE rebrickable_inventory_parts; + TRUNCATE TABLE rebrickable_inventory_sets; + TRUNCATE TABLE rebrickable_inventory; + TRUNCATE TABLE rebrickable_set; + TRUNCATE TABLE rebrickable_theme; + TRUNCATE TABLE rebrickable_part; + TRUNCATE TABLE rebrickable_category; + TRUNCATE TABLE rebrickable_color; + '; + + return $this->em->getConnection()->prepare($query)->execute(); + } + + private function loadCsvFile($file, $table, $columns) + { + $query = sprintf("LOAD DATA LOCAL INFILE '%s' + REPLACE INTO TABLE %s + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' + LINES TERMINATED BY '\\n' + IGNORE 1 LINES %s", addslashes($file), $table, $columns); + + return $this->em->getConnection()->prepare($query)->execute(); + } + + private function addMissingParts() + { + $connection = $this->em->getConnection(); + $statement = $connection->prepare( + 'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts + LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id + WHERE rebrickable_part.id IS NULL'); + $statement->execute(); + $foreignKeys = $statement->fetchAll(); + + foreach ($foreignKeys as $foreignKey) { + $part = new Part(); + $part->setNumber($foreignKey['part_id']); + $this->em->getRepository(Part::class)->save($part); + } + } + + private function loadInventoryTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_inventory', '(`id`,`version`,`set_id`)'); + } + + private function loadInventoryPartTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)'); + } + + private function loadInventorySetTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)'); + } + + private function loadSetTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)'); + } + + private function loadThemeTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')'); + } + + private function loadPartTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_part', '(`id`,`name`,`category_id`)'); + } + + private function loadCategoryTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_category', '(`id`,`name`)'); + } + + private function loadColorTable($csv) + { + return $this->loadCsvFile($csv, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)'); + } +} diff --git a/src/AppBundle/Service/Loader/RebrickableLoaderService.php b/src/AppBundle/Service/Loader/RebrickableLoaderService.php deleted file mode 100644 index 65bfa34..0000000 --- a/src/AppBundle/Service/Loader/RebrickableLoaderService.php +++ /dev/null @@ -1,152 +0,0 @@ -rebrickable_url = $rebrickable_url; - } - - public function loadTables() - { - $connection = $this->em->getConnection(); - - try { - $connection->beginTransaction(); - $connection->prepare('SET foreign_key_checks = 0;')->execute(); - $this->truncateTables(); - $connection->prepare('SET foreign_key_checks = 1;')->execute(); - - $this->loadColorTable(); - $this->loadCategoryTable(); - $this->loadPartTable(); - $this->loadThemeTable(); - $this->loadSetTable(); - $this->loadInventoryTable(); - $this->loadInventorySetTable(); - - $connection->prepare('SET foreign_key_checks = 0;')->execute(); - $this->loadInventoryPartTable(); - $connection->prepare('SET foreign_key_checks = 1;')->execute(); - $this->addMissingParts(); - - $connection->commit(); - } catch (\Exception $e) { - $connection->rollBack(); - throw $e; - } - } - - private function truncateTables() - { - $query =' - TRUNCATE TABLE rebrickable_inventory_parts; - TRUNCATE TABLE rebrickable_inventory_sets; - TRUNCATE TABLE rebrickable_color; - TRUNCATE TABLE rebrickable_category; - TRUNCATE TABLE rebrickable_inventory; - TRUNCATE TABLE rebrickable_set; - TRUNCATE TABLE rebrickable_theme; - TRUNCATE TABLE rebrickable_part; - '; - - return $this->em->getConnection()->prepare($query)->execute(); - } - - private function loadCsvFile($file, $table, $columns) - { - $query = sprintf("LOAD DATA LOCAL INFILE '%s' - REPLACE INTO TABLE %s - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' - LINES TERMINATED BY '\\n' - IGNORE 1 LINES %s", addslashes($file), $table, $columns); - - return $this->em->getConnection()->prepare($query)->execute(); - } - - private function addMissingParts() { - $connection = $this->em->getConnection(); - $statement = $connection->prepare( - 'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts - LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id - WHERE rebrickable_part.id IS NULL'); - $statement->execute(); - $foreignKeys = $statement->fetchAll(); - - foreach ($foreignKeys as $foreignKey) { - $part = new Part(); - $part->setNumber($foreignKey['part_id']); - $this->em->getRepository(Part::class)->save($part); - } - } - - private function loadInventoryTable() - { - $file = $this->downloadFile($this->rebrickable_url.'inventories.csv'); - - return $this->loadCsvFile($file, 'rebrickable_inventory', '(`id`,`version`,`set_id`)'); - } - - private function loadInventoryPartTable() - { - $file = $this->downloadFile($this->rebrickable_url.'inventory_parts.csv'); - - return $this->loadCsvFile($file, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)'); - } - - private function loadInventorySetTable() - { - $file = $this->downloadFile($this->rebrickable_url.'inventory_sets.csv'); - - return $this->loadCsvFile($file, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)'); - } - - private function loadSetTable() - { - $file = $this->downloadFile($this->rebrickable_url.'sets.csv'); - - return $this->loadCsvFile($file, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)'); - } - - private function loadThemeTable() - { - $file = $this->downloadFile($this->rebrickable_url.'themes.csv'); - - return $this->loadCsvFile($file, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')'); - } - - private function loadPartTable() - { - $file = $this->downloadFile($this->rebrickable_url.'parts.csv'); - - return $this->loadCsvFile($file, 'rebrickable_part', '(`id`,`name`,`category_id`)'); - } - - private function loadCategoryTable() - { - $file = $this->downloadFile($this->rebrickable_url.'part_categories.csv'); - - return $this->loadCsvFile($file, 'rebrickable_category', '(`id`,`name`)'); - } - - private function loadColorTable() - { - $file = $this->downloadFile($this->rebrickable_url.'colors.csv'); - - return $this->loadCsvFile($file, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)'); - } -}