Improve load rebrickable data commad

app/config/config.yml

@@ -7,8 +7,8 @@ imports:
 # http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
 parameters:
     locale: en
-    # rebrickable csv files root URL
+    # rebrickable csv files root URL (http://rebrickable.com/media/downloads/ or local dir containing csv files)
-    rebrickable_url: 'http://rebrickable.com/media/downloads/'
+    rebrickable_csv_url: 'http://rebrickable.com/media/downloads/'
 
 framework:
     #esi:             ~

src/AppBundle/Command/LoadRebrickableDataCommand.php

@@ -12,8 +12,8 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
     {
         $this
             ->setName('app:load:rebrickable')
-            ->setDescription('Loads Rebrickable database')
+            ->setDescription('Loads Rebrickable data about sets and parts into database.')
-            ->setHelp('This command allows you to..');
+            ->setHelp('This command allows you to load Rebrickable CSV files containing information about sets and parts into database.');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)

src/AppBundle/Service/Loader/RebrickableLoader.php

@@ -0,0 +1,166 @@
+<?php
+
+namespace AppBundle\Service\Loader;
+
+use AppBundle\Entity\Rebrickable\Part;
+use AppBundle\Entity\Rebrickable\Set;
+use Doctrine\ORM\Query\Expr\Join;
+
+//TODO Refactor + validate csv files
+class RebrickableLoader extends BaseLoader
+{
+    private $rebrickable_url;
+
+    private $csvFile;
+
+    /**
+     * ModelLoaderService constructor.
+     */
+    public function __construct($rebrickable_url)
+    {
+        $this->rebrickable_url = $rebrickable_url;
+    }
+
+    public function loadTables()
+    {
+        $connection = $this->em->getConnection();
+        $connection->beginTransaction();
+
+        try {
+            $this->loadCsvFiles();
+
+            $connection->prepare('SET foreign_key_checks = 0;')->execute();
+            $this->truncateTables();
+            $connection->prepare('SET foreign_key_checks = 1;')->execute();
+
+            $this->output->writeln([
+                '<info>Truncated</info> <comment>rebrickable</comment> <info>database tables.</info>',
+                '------------------------------------------------------------------------------',
+                'Loading CSV files into database...',
+            ]);
+
+            $this->loadColorTable($this->csvFile['colors']);
+            $this->loadCategoryTable($this->csvFile['part_categories']);
+            $this->loadPartTable($this->csvFile['parts']);
+            $this->loadThemeTable($this->csvFile['themes']);
+            $this->loadSetTable($this->csvFile['sets']);
+            $this->loadInventoryTable($this->csvFile['inventories']);
+            $this->loadInventorySetTable($this->csvFile['inventory_sets']);
+
+            $connection->prepare('SET foreign_key_checks = 0;')->execute();
+            $this->loadInventoryPartTable($this->csvFile['inventory_parts']);
+            $connection->prepare('SET foreign_key_checks = 1;')->execute();
+            $this->addMissingParts();
+
+            $connection->commit();
+
+            $this->output->writeln('Rebrickable database loaded successfully!');
+        } catch (\Exception $e) {
+            $connection->rollBack();
+            throw $e;
+        }
+    }
+
+    private function loadCSVFiles()
+    {
+        $array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
+
+        $this->output->writeln([
+            'Loading Rebrickable CSV files',
+            '------------------------------------------------------------------------------',
+        ]);
+
+        foreach ($array as $item) {
+            $this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
+        }
+
+        $this->output->writeln([
+            '<info>Done</info>',
+            '------------------------------------------------------------------------------',
+        ]);
+    }
+
+    private function truncateTables()
+    {
+        $query = '
+            TRUNCATE TABLE rebrickable_inventory_parts;
+            TRUNCATE TABLE rebrickable_inventory_sets;
+            TRUNCATE TABLE rebrickable_inventory;
+            TRUNCATE TABLE rebrickable_set;
+            TRUNCATE TABLE rebrickable_theme;
+            TRUNCATE TABLE rebrickable_part;
+            TRUNCATE TABLE rebrickable_category;
+            TRUNCATE TABLE rebrickable_color;
+           ';
+
+        return $this->em->getConnection()->prepare($query)->execute();
+    }
+
+    private function loadCsvFile($file, $table, $columns)
+    {
+        $query = sprintf("LOAD DATA LOCAL INFILE '%s' 
+            REPLACE INTO TABLE %s
+            FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
+            LINES TERMINATED BY '\\n'
+            IGNORE 1 LINES %s", addslashes($file), $table, $columns);
+
+        return $this->em->getConnection()->prepare($query)->execute();
+    }
+
+    private function addMissingParts()
+    {
+        $connection = $this->em->getConnection();
+        $statement = $connection->prepare(
+            'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
+                 LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
+                 WHERE rebrickable_part.id IS NULL');
+        $statement->execute();
+        $foreignKeys = $statement->fetchAll();
+
+        foreach ($foreignKeys as $foreignKey) {
+            $part = new Part();
+            $part->setNumber($foreignKey['part_id']);
+            $this->em->getRepository(Part::class)->save($part);
+        }
+    }
+
+    private function loadInventoryTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
+    }
+
+    private function loadInventoryPartTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
+    }
+
+    private function loadInventorySetTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
+    }
+
+    private function loadSetTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
+    }
+
+    private function loadThemeTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
+    }
+
+    private function loadPartTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_part', '(`id`,`name`,`category_id`)');
+    }
+
+    private function loadCategoryTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_category', '(`id`,`name`)');
+    }
+
+    private function loadColorTable($csv)
+    {
+        return $this->loadCsvFile($csv, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
+    }
+}

src/AppBundle/Service/Loader/RebrickableLoaderService.php

@@ -1,152 +0,0 @@
-<?php
-
-namespace AppBundle\Service\Loader;
-
-use AppBundle\Entity\Rebrickable\Inventory_Part;
-use AppBundle\Entity\Rebrickable\Inventory_Set;
-use AppBundle\Entity\Rebrickable\Part;
-use AppBundle\Entity\Rebrickable\Set;
-use Doctrine\ORM\Query\Expr\Join;
-
-//TODO Refactor + validate csv files
-class RebrickableLoaderService extends BaseLoaderService
-{
-    private $rebrickable_url;
-
-    /**
-     * ModelLoaderService constructor.
-     */
-    public function __construct($rebrickable_url)
-    {
-        $this->rebrickable_url = $rebrickable_url;
-    }
-
-    public function loadTables()
-    {
-        $connection = $this->em->getConnection();
-
-        try {
-            $connection->beginTransaction();
-            $connection->prepare('SET foreign_key_checks = 0;')->execute();
-            $this->truncateTables();
-            $connection->prepare('SET foreign_key_checks = 1;')->execute();
-
-            $this->loadColorTable();
-            $this->loadCategoryTable();
-            $this->loadPartTable();
-            $this->loadThemeTable();
-            $this->loadSetTable();
-            $this->loadInventoryTable();
-            $this->loadInventorySetTable();
-
-            $connection->prepare('SET foreign_key_checks = 0;')->execute();
-            $this->loadInventoryPartTable();
-            $connection->prepare('SET foreign_key_checks = 1;')->execute();
-            $this->addMissingParts();
-
-            $connection->commit();
-        } catch (\Exception $e) {
-            $connection->rollBack();
-            throw $e;
-        }
-    }
-
-    private function truncateTables()
-    {
-        $query ='
-            TRUNCATE TABLE rebrickable_inventory_parts;
-            TRUNCATE TABLE rebrickable_inventory_sets;
-            TRUNCATE TABLE rebrickable_color;
-            TRUNCATE TABLE rebrickable_category;
-            TRUNCATE TABLE rebrickable_inventory;
-            TRUNCATE TABLE rebrickable_set;
-            TRUNCATE TABLE rebrickable_theme;
-            TRUNCATE TABLE rebrickable_part;
-           ';
-
-        return $this->em->getConnection()->prepare($query)->execute();
-    }
-
-    private function loadCsvFile($file, $table, $columns)
-    {
-        $query = sprintf("LOAD DATA LOCAL INFILE '%s' 
-            REPLACE INTO TABLE %s
-            FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
-            LINES TERMINATED BY '\\n'
-            IGNORE 1 LINES %s", addslashes($file), $table, $columns);
-
-        return $this->em->getConnection()->prepare($query)->execute();
-    }
-
-    private function addMissingParts() {
-        $connection = $this->em->getConnection();
-        $statement = $connection->prepare(
-            'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
-                 LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
-                 WHERE rebrickable_part.id IS NULL');
-        $statement->execute();
-        $foreignKeys = $statement->fetchAll();
-
-        foreach ($foreignKeys as $foreignKey) {
-            $part = new Part();
-            $part->setNumber($foreignKey['part_id']);
-            $this->em->getRepository(Part::class)->save($part);
-        }
-    }
-
-    private function loadInventoryTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'inventories.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
-    }
-
-    private function loadInventoryPartTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'inventory_parts.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
-    }
-
-    private function loadInventorySetTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'inventory_sets.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
-    }
-
-    private function loadSetTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'sets.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
-    }
-
-    private function loadThemeTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'themes.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
-    }
-
-    private function loadPartTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'parts.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_part', '(`id`,`name`,`category_id`)');
-    }
-
-    private function loadCategoryTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'part_categories.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_category', '(`id`,`name`)');
-    }
-
-    private function loadColorTable()
-    {
-        $file = $this->downloadFile($this->rebrickable_url.'colors.csv');
-
-        return $this->loadCsvFile($file, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
-    }
-}