diff --git a/app/config/config.yml b/app/config/config.yml
index c6f65ad..9bf88c9 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -7,8 +7,8 @@ imports:
# http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
parameters:
locale: en
- # rebrickable csv files root URL
- rebrickable_url: 'http://rebrickable.com/media/downloads/'
+ # rebrickable csv files root URL (http://rebrickable.com/media/downloads/ or local dir containing csv files)
+ rebrickable_csv_url: 'http://rebrickable.com/media/downloads/'
framework:
#esi: ~
diff --git a/src/AppBundle/Command/LoadRebrickableDataCommand.php b/src/AppBundle/Command/LoadRebrickableDataCommand.php
index a9c441e..71d7135 100644
--- a/src/AppBundle/Command/LoadRebrickableDataCommand.php
+++ b/src/AppBundle/Command/LoadRebrickableDataCommand.php
@@ -12,8 +12,8 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
{
$this
->setName('app:load:rebrickable')
- ->setDescription('Loads Rebrickable database')
- ->setHelp('This command allows you to..');
+ ->setDescription('Loads Rebrickable data about sets and parts into database.')
+ ->setHelp('This command allows you to load Rebrickable CSV files containing information about sets and parts into database.');
}
protected function execute(InputInterface $input, OutputInterface $output)
diff --git a/src/AppBundle/Service/Loader/RebrickableLoader.php b/src/AppBundle/Service/Loader/RebrickableLoader.php
new file mode 100644
index 0000000..7df7f91
--- /dev/null
+++ b/src/AppBundle/Service/Loader/RebrickableLoader.php
@@ -0,0 +1,166 @@
+rebrickable_url = $rebrickable_url;
+ }
+
+ public function loadTables()
+ {
+ $connection = $this->em->getConnection();
+ $connection->beginTransaction();
+
+ try {
+ $this->loadCsvFiles();
+
+ $connection->prepare('SET foreign_key_checks = 0;')->execute();
+ $this->truncateTables();
+ $connection->prepare('SET foreign_key_checks = 1;')->execute();
+
+ $this->output->writeln([
+ 'Truncated rebrickable database tables.',
+ '------------------------------------------------------------------------------',
+ 'Loading CSV files into database...',
+ ]);
+
+ $this->loadColorTable($this->csvFile['colors']);
+ $this->loadCategoryTable($this->csvFile['part_categories']);
+ $this->loadPartTable($this->csvFile['parts']);
+ $this->loadThemeTable($this->csvFile['themes']);
+ $this->loadSetTable($this->csvFile['sets']);
+ $this->loadInventoryTable($this->csvFile['inventories']);
+ $this->loadInventorySetTable($this->csvFile['inventory_sets']);
+
+ $connection->prepare('SET foreign_key_checks = 0;')->execute();
+ $this->loadInventoryPartTable($this->csvFile['inventory_parts']);
+ $connection->prepare('SET foreign_key_checks = 1;')->execute();
+ $this->addMissingParts();
+
+ $connection->commit();
+
+ $this->output->writeln('Rebrickable database loaded successfully!');
+ } catch (\Exception $e) {
+ $connection->rollBack();
+ throw $e;
+ }
+ }
+
+ private function loadCSVFiles()
+ {
+ $array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
+
+ $this->output->writeln([
+ 'Loading Rebrickable CSV files',
+ '------------------------------------------------------------------------------',
+ ]);
+
+ foreach ($array as $item) {
+ $this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
+ }
+
+ $this->output->writeln([
+ 'Done',
+ '------------------------------------------------------------------------------',
+ ]);
+ }
+
+ private function truncateTables()
+ {
+ $query = '
+ TRUNCATE TABLE rebrickable_inventory_parts;
+ TRUNCATE TABLE rebrickable_inventory_sets;
+ TRUNCATE TABLE rebrickable_inventory;
+ TRUNCATE TABLE rebrickable_set;
+ TRUNCATE TABLE rebrickable_theme;
+ TRUNCATE TABLE rebrickable_part;
+ TRUNCATE TABLE rebrickable_category;
+ TRUNCATE TABLE rebrickable_color;
+ ';
+
+ return $this->em->getConnection()->prepare($query)->execute();
+ }
+
+ private function loadCsvFile($file, $table, $columns)
+ {
+ $query = sprintf("LOAD DATA LOCAL INFILE '%s'
+ REPLACE INTO TABLE %s
+ FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
+ LINES TERMINATED BY '\\n'
+ IGNORE 1 LINES %s", addslashes($file), $table, $columns);
+
+ return $this->em->getConnection()->prepare($query)->execute();
+ }
+
+ private function addMissingParts()
+ {
+ $connection = $this->em->getConnection();
+ $statement = $connection->prepare(
+ 'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
+ LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
+ WHERE rebrickable_part.id IS NULL');
+ $statement->execute();
+ $foreignKeys = $statement->fetchAll();
+
+ foreach ($foreignKeys as $foreignKey) {
+ $part = new Part();
+ $part->setNumber($foreignKey['part_id']);
+ $this->em->getRepository(Part::class)->save($part);
+ }
+ }
+
+ private function loadInventoryTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
+ }
+
+ private function loadInventoryPartTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
+ }
+
+ private function loadInventorySetTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
+ }
+
+ private function loadSetTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
+ }
+
+ private function loadThemeTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
+ }
+
+ private function loadPartTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_part', '(`id`,`name`,`category_id`)');
+ }
+
+ private function loadCategoryTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_category', '(`id`,`name`)');
+ }
+
+ private function loadColorTable($csv)
+ {
+ return $this->loadCsvFile($csv, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
+ }
+}
diff --git a/src/AppBundle/Service/Loader/RebrickableLoaderService.php b/src/AppBundle/Service/Loader/RebrickableLoaderService.php
deleted file mode 100644
index 65bfa34..0000000
--- a/src/AppBundle/Service/Loader/RebrickableLoaderService.php
+++ /dev/null
@@ -1,152 +0,0 @@
-rebrickable_url = $rebrickable_url;
- }
-
- public function loadTables()
- {
- $connection = $this->em->getConnection();
-
- try {
- $connection->beginTransaction();
- $connection->prepare('SET foreign_key_checks = 0;')->execute();
- $this->truncateTables();
- $connection->prepare('SET foreign_key_checks = 1;')->execute();
-
- $this->loadColorTable();
- $this->loadCategoryTable();
- $this->loadPartTable();
- $this->loadThemeTable();
- $this->loadSetTable();
- $this->loadInventoryTable();
- $this->loadInventorySetTable();
-
- $connection->prepare('SET foreign_key_checks = 0;')->execute();
- $this->loadInventoryPartTable();
- $connection->prepare('SET foreign_key_checks = 1;')->execute();
- $this->addMissingParts();
-
- $connection->commit();
- } catch (\Exception $e) {
- $connection->rollBack();
- throw $e;
- }
- }
-
- private function truncateTables()
- {
- $query ='
- TRUNCATE TABLE rebrickable_inventory_parts;
- TRUNCATE TABLE rebrickable_inventory_sets;
- TRUNCATE TABLE rebrickable_color;
- TRUNCATE TABLE rebrickable_category;
- TRUNCATE TABLE rebrickable_inventory;
- TRUNCATE TABLE rebrickable_set;
- TRUNCATE TABLE rebrickable_theme;
- TRUNCATE TABLE rebrickable_part;
- ';
-
- return $this->em->getConnection()->prepare($query)->execute();
- }
-
- private function loadCsvFile($file, $table, $columns)
- {
- $query = sprintf("LOAD DATA LOCAL INFILE '%s'
- REPLACE INTO TABLE %s
- FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
- LINES TERMINATED BY '\\n'
- IGNORE 1 LINES %s", addslashes($file), $table, $columns);
-
- return $this->em->getConnection()->prepare($query)->execute();
- }
-
- private function addMissingParts() {
- $connection = $this->em->getConnection();
- $statement = $connection->prepare(
- 'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
- LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
- WHERE rebrickable_part.id IS NULL');
- $statement->execute();
- $foreignKeys = $statement->fetchAll();
-
- foreach ($foreignKeys as $foreignKey) {
- $part = new Part();
- $part->setNumber($foreignKey['part_id']);
- $this->em->getRepository(Part::class)->save($part);
- }
- }
-
- private function loadInventoryTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'inventories.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
- }
-
- private function loadInventoryPartTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'inventory_parts.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
- }
-
- private function loadInventorySetTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'inventory_sets.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
- }
-
- private function loadSetTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'sets.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
- }
-
- private function loadThemeTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'themes.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
- }
-
- private function loadPartTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'parts.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_part', '(`id`,`name`,`category_id`)');
- }
-
- private function loadCategoryTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'part_categories.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_category', '(`id`,`name`)');
- }
-
- private function loadColorTable()
- {
- $file = $this->downloadFile($this->rebrickable_url.'colors.csv');
-
- return $this->loadCsvFile($file, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
- }
-}