Download rebrickable csv with command

src/AppBundle/Command/LoadRebrickableCommand.php

@@ -3,7 +3,6 @@
 namespace AppBundle\Command;
 
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
-use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
@@ -14,10 +13,7 @@ class LoadRebrickableCommand extends ContainerAwareCommand
         $this
             ->setName('app:loadRebrickable')
             ->setDescription('Loads Rebrickable csv data')
-            ->setHelp("This command allows you to..")
+            ->setHelp('This command allows you to..')
-            ->addArgument('pieces', InputArgument::REQUIRED, 'Path to Rebrickable pieces.csv file')
-            ->addArgument('sets', InputArgument::REQUIRED, 'Path to Rebrickable sets.csv file')
-            ->addArgument('set_pieces', InputArgument::REQUIRED, 'Path to Rebrickable set_pieces.csv file')
         ;
     }
 
@@ -25,13 +21,16 @@ class LoadRebrickableCommand extends ContainerAwareCommand
     {
         $modelLoader = $this->getContainer()->get('app.model_loader_service');
 
-        printf('colors'."\n");
+        try {
             $modelLoader->loadColors();
-        printf('sets'."\n");
+
-        $modelLoader->loadBuildingKits(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('sets'));
+            $modelLoader->loadParts($output);
-        printf('pieces'."\n");
+
-        $modelLoader->loadParts(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('pieces'));
+            $modelLoader->loadBuildingKits($output);
-        printf('set_pieces'."\n");
+
-        $modelLoader->loadPartBuildingKits(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('set_pieces'));
+            $modelLoader->loadPartBuildingKits($output);
+        } catch (\Exception $e) {
+            printf($e->getMessage());
+        }
     }
 }

src/AppBundle/Service/ModelLoaderService.php

@@ -12,6 +12,7 @@ use AppBundle\Entity\Part;
 use AppBundle\Entity\Part_BuildingKit;
 use Doctrine\ORM\EntityManager;
 use Symfony\Component\Asset\Exception\LogicException;
+use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Finder\Finder;
 use Symfony\Component\Finder\SplFileInfo;
 
@@ -92,26 +93,30 @@ class ModelLoaderService
 
     // Rebrickable
 
-    public function loadPartBuildingKits($set_pieces) {
+    public function loadPartBuildingKits($output)
+    {
         $partRepository = $this->em->getRepository('AppBundle:Part');
         $buldingKitRepository = $this->em->getRepository('AppBundle:BuildingKit');
         $colorRepository = $this->em->getRepository('AppBundle:Color');
 
+        $setPieces = tempnam(sys_get_temp_dir(), 'printabrick.');
+        file_put_contents($setPieces, fopen('compress.zlib://http://rebrickable.com/files/set_pieces.csv.gz', 'r'));
+
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
 
-        if (($handle = fopen($set_pieces, "r")) !== FALSE) {
+        if (($handle = fopen($setPieces, 'r')) !== false) {
-            $header = fgetcsv($handle, 200, ",");
+            $header = fgetcsv($handle, 200, ',');
-            $line = 0;
+
-            while (($data = fgetcsv($handle, 200, ",")) !== FALSE) {
+            // create a new progress bar (50 units)
+            $progress = new ProgressBar($output, intval(exec("wc -l '$setPieces'")));
+            $progress->setFormat('very_verbose');
+            $progress->setBarWidth(50);
+            $progress->start();
+
+            $index = 0;
+            while (($data = fgetcsv($handle, 200, ',')) !== false) {
                 $partBuildingKit = new Part_BuildingKit();
 
-                $line = $line + 1;
-
-                if($line%1000 == 0 ) {
-                    var_dump($line);
-                    $this->em->flush();
-                }
-
                 $buildingKit = $buldingKitRepository->findOneBy(['number' => $data[0]]);
                 $part = $partRepository->findOneBy(['number' => $data[1]]);
                 $color = $colorRepository->findOneBy(['id' => $data[3]]);
@@ -126,23 +131,46 @@ class ModelLoaderService
 
                     $this->em->persist($partBuildingKit);
                 }
+
+                $index = $index + 1;
+                if ($index % 25 == 0) {
+                    $this->em->flush();
+                    $this->em->clear();
+                }
+                $progress->advance();
             }
 
             $this->em->flush();
+            $this->em->clear();
             fclose($handle);
-        }
+            $progress->finish();
+            $progress->clear();
         }
 
+        unlink($setPieces);
+    }
 
-    public function loadBuildingKits($sets_file) {
+    public function loadBuildingKits($output)
+    {
         $keywordRepository = $this->em->getRepository('AppBundle:Keyword');
 
-        if (($handle = fopen($sets_file, "r")) !== FALSE) {
+        $sets = tempnam(sys_get_temp_dir(), 'printabrick.');
-            $header = fgetcsv($handle, 1000, ",");
+        file_put_contents($sets, fopen('compress.zlib://http://rebrickable.com/files/sets.csv.gz', 'r'));
-            while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
+
+        if (($handle = fopen($sets, 'r')) !== false) {
+            $header = fgetcsv($handle, 500, ',');
+
+            // create a new progress bar (50 units)
+            $progress = new ProgressBar($output, intval(exec("wc -l '$sets'")));
+            $progress->setFormat('very_verbose');
+            $progress->setBarWidth(50);
+            $progress->start();
+
+            $index = 0;
+            while (($data = fgetcsv($handle, 500, ',')) !== false) {
                 $buildingKit = new BuildingKit();
 
-                for($i=3; $i <=5; $i++) {
+                for ($i = 3; $i <= 5; ++$i) {
                     $keyword = $keywordRepository->findOneBy(['name' => $data[$i]]);
                     if ($keyword == null) {
                         $keyword = new Keyword();
@@ -154,7 +182,6 @@ class ModelLoaderService
                     $buildingKit->addKeyword($keyword);
                 }
 
-
                 $buildingKit
                     ->setNumber($data[0])
                     ->setYear($data[1])
@@ -163,23 +190,45 @@ class ModelLoaderService
 
                 $this->em->persist($buildingKit);
 
+                $index = $index + 1;
+                if ($index % 25 == 0) {
+                    $this->em->flush();
+                    $this->em->clear();
+                }
+
+                $progress->advance();
             }
             $this->em->flush();
+            $this->em->clear();
 
             fclose($handle);
+
+            $progress->finish();
+            $progress->clear();
         }
+        unlink($sets);
     }
 
-    public function loadParts($pieces_file) {
+    public function loadParts($output)
+    {
+        $pieces = tempnam(sys_get_temp_dir(), 'printabrick.');
+        file_put_contents($pieces, fopen('compress.zlib://http://rebrickable.com/files/pieces.csv.gz', 'r'));
+
         $categoryRepository = $this->em->getRepository('AppBundle:Category');
 
-        if (($handle = fopen($pieces_file, "r")) !== FALSE) {
+        if (($handle = fopen($pieces, 'r')) !== false) {
-            $header = fgetcsv($handle, 300, ",");
+
-            while (($data = fgetcsv($handle, 300, ",")) !== FALSE) {
+            // create a new progress bar (50 units)
+            $progress = new ProgressBar($output, intval(exec("wc -l '$pieces'")));
+            $progress->setFormat('very_verbose');
+            $progress->setBarWidth(50);
+            $progress->start();
+
+            $header = fgetcsv($handle, 300, ',');
+            while (($data = fgetcsv($handle, 300, ',')) !== false) {
                 $part = new Part();
                 $part->setNumber($data[0])->setName($data[1]);
 
-
                 $category = $categoryRepository->findOneBy(['name' => $data[2]]);
                 if ($category == null) {
                     $category = new Category();
@@ -194,16 +243,23 @@ class ModelLoaderService
 
                 $this->em->persist($part);
 
-//                $this->em->persist($category);
+                $progress->advance();
             }
 
             $this->em->flush();
+            $this->em->clear();
 
             fclose($handle);
-        }
+
+            $progress->finish();
+            $progress->clear();
         }
 
-    public function loadColors() {
+        unlink($pieces);
+    }
+
+    public function loadColors()
+    {
         $rb_colors = $this->rebrickableManager->getColors();
 
         foreach ($rb_colors as $rb_color) {
@@ -219,7 +275,8 @@ class ModelLoaderService
         $this->em->flush();
     }
 
-    public function getModel(Part $part) {
+    public function getModel(Part $part)
+    {
         $modelRepository = $this->em->getRepository('AppBundle:Model');
 
         if (strpos($part->getNumber(), 'p')) {