Setup download urls in config.yml

app/config/config.yml

@@ -7,6 +7,11 @@ imports:
 # http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
 parameters:
     locale: en
+    rebrickable_url:
+      pieces: 'http://rebrickable.com/files/pieces.csv.gz'
+      sets: 'http://rebrickable.com/files/sets.csv.gz'
+      set_pieces: 'http://rebrickable.com/files/set_pieces.csv.gz'
+    ldraw_url: 'http://www.ldraw.org/library/updates/completeCA.zip'
 
 framework:
     #esi:             ~

app/config/services.yml

@@ -18,11 +18,11 @@ services:
         arguments: ['@doctrine.orm.entity_manager', '@manager.brickset','@manager.rebrickable']
 
     loader.rebrickable:
-        class: AppBundle\Loader\RebrickableLoader
+        class: AppBundle\Command\Loader\RebrickableLoader
-        arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable']
+        arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable', '%rebrickable_url%' ]
     loader.ldraw:
-        class: AppBundle\Loader\LDrawLoader
+        class: AppBundle\Command\Loader\LDrawLoader
-        arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem']
+        arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem', '%ldraw_url%']
   
     app.form.filter_set:
         class: AppBundle\Form\FilterSetType

src/AppBundle/Command/LoadLDrawCommand.php

@@ -1,29 +0,0 @@
-<?php
-
-namespace AppBundle\Command;
-
-use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
-use Symfony\Component\Console\Input\InputArgument;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Output\OutputInterface;
-
-class LoadLDrawCommand extends ContainerAwareCommand
-{
-    protected function configure()
-    {
-        $this
-            ->setName('app:load:ldraw')
-            ->setDescription('Loads LDraw library parts')
-            ->setHelp('This command allows you to..')
-            ->addArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library folder');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {
-        $ldrawLoader = $this->getContainer()->get('loader.ldraw');
-
-        $ldrawLoader->setOutput($output);
-
-        $ldrawLoader->loadModels($input->getArgument('ldraw'));
-    }
-}

src/AppBundle/Command/LoadLibraryCommand.php

@@ -3,26 +3,32 @@
 namespace AppBundle\Command;
 
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
-class LoadRebrickableCommand extends ContainerAwareCommand
+class LoadLibraryCommand extends ContainerAwareCommand
 {
     protected function configure()
     {
         $this
-            ->setName('app:load:rebrickable')
+            ->setName('app:load:library')
-            ->setDescription('Loads Rebrickable csv data')
+            ->setDescription('Loads LDraw library parts')
-            ->setHelp('This command allows you to..');
+            ->setHelp('This command allows you to..')
+            ->addArgument('ldraw', InputArgument::OPTIONAL, 'Path to LDraw library folder');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $rebrickableLoader = $this->getContainer()->get('loader.rebrickable');
+        $ldrawLoader = $this->getContainer()->get('loader.ldraw');
+        $ldrawLoader->setOutput($output);
 
+        $rebrickableLoader = $this->getContainer()->get('loader.rebrickable');
         $rebrickableLoader->setOutput($output);
 
         try {
+            $ldrawLoader->loadModels($input->getArgument('ldraw'));
+
             $rebrickableLoader->loadColors();
 
             $rebrickableLoader->loadParts();

src/AppBundle/Command/Loader/LDrawLoader.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace AppBundle\Loader;
+namespace AppBundle\Command\Loader;
 
 use AppBundle\Entity\Category;
 use AppBundle\Entity\Model;
@@ -10,7 +10,6 @@ use Symfony\Component\Asset\Exception\LogicException;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Finder\Finder;
 use Symfony\Component\Finder\SplFileInfo;
-use Symfony\Component\Process\Exception\ProcessFailedException;
 use Symfony\Component\Process\ProcessBuilder;
 
 class LDrawLoader extends Loader
@@ -30,7 +29,9 @@ class LDrawLoader extends Loader
      */
     private $dataPath;
 
-    public function __construct($em, $ldview, $dataPath)
+    private $ldraw_url;
+
+    public function __construct($em, $ldview, $dataPath, $ldraw_url)
     {
         /*
          * @var $em EntityManager
@@ -38,6 +39,27 @@ class LDrawLoader extends Loader
         $this->em = $em;
         $this->ldview = $ldview;
         $this->dataPath = $dataPath;
+        $this->ldraw_url = $ldraw_url;
+    }
+
+    public function downloadLibrary()
+    {
+        $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com');
+        $temp = $this->downloadFile($this->ldraw_url);
+        $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.');
+        if (file_exists($temp_dir)) {
+            unlink($temp_dir);
+        }
+        mkdir($temp_dir);
+        $zip = new \ZipArchive();
+        if ($zip->open($temp) != 'true') {
+            echo 'Error :- Unable to open the Zip File';
+        }
+        $zip->extractTo($temp_dir);
+        $zip->close();
+        unlink($temp);
+
+        return $temp_dir;
     }
 
     public function loadModels($LDrawLibrary)
@@ -85,9 +107,16 @@ class LDrawLoader extends Loader
                 if ($line !== '') {
                     $line = preg_replace('/^0 /', '', $line);
 
-                    // 0 <PartDescription>
+                    // 0 <CategoryName> <PartDescription>
-                    if (!$firstLine) { //TODO handle ~Moved to
+                    if (!$firstLine) {
-                        $category = explode(' ', trim($line))[0];
+                        //TODO handle "~Moved to"
+                        //TODO        "=" - alias name for other part kept for referece
+                        //TODO        "_" shortcut
+
+                        $array = explode(' ', trim($line), 2);
+                        $category = isset($array[0]) ? $array[0] : '';
+                        $model->setName($line);
+
                         $firstLine = true;
                     }
                     // 0 !CATEGORY <CategoryName>
@@ -113,8 +142,6 @@ class LDrawLoader extends Loader
             if ($cat == null) {
                 $cat = new Category();
                 $cat->setName($category);
-                $this->em->persist($cat);
-                $this->em->flush();
             }
 
             $model->setCategory($cat);
@@ -134,25 +161,25 @@ class LDrawLoader extends Loader
      */
     private function createStlFile($file)
     {
-        $builder = new ProcessBuilder();
-        $process = $builder
-            ->setPrefix($this->ldview)
-            ->setArguments([
-//                $this->ldraw->getAdapter()->getPathPrefix().$file['path'],
-                $file->getRealPath(),
-                '-ExportFiles=1',
-                '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(),
-                '-ExportSuffix=.stl',
-                '-ExportsDir='.$this->dataPath->getAdapter()->getPathPrefix(),
-            ])
-            ->getProcess();
-
-        $process->run();
-
         $stlFilename = str_replace('.dat', '.stl', $file->getFilename());
 
-        if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) {
+        if (!$this->dataPath->has($stlFilename)) {
-            throw new ProcessFailedException($process); //TODO
+            $builder = new ProcessBuilder();
+            $process = $builder
+                ->setPrefix($this->ldview)
+                ->setArguments([
+//                $this->ldraw->getAdapter()->getPathPrefix().$file['path'],
+                    $file->getRealPath(),
+                    '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(),
+                    '-ExportFile='.$this->dataPath->getAdapter()->getPathPrefix().$stlFilename,
+                ])
+                ->getProcess();
+
+            $process->run();
+
+            if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) {
+                throw new LogicException($file->getFilename().' : '.$process->getOutput()); //TODO
+            }
         }
 
         return $this->dataPath->get($stlFilename);

src/AppBundle/Command/Loader/Loader.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace AppBundle\Command\Loader;
+
+use Doctrine\ORM\EntityManager;
+use Symfony\Component\Asset\Exception\LogicException;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Debug\Exception\ContextErrorException;
+
+class Loader
+{
+    /**
+     * @var EntityManager
+     */
+    protected $em;
+
+    /**
+     * @var OutputInterface
+     */
+    protected $output;
+
+    /**
+     * @var ProgressBar
+     */
+    protected $progressBar;
+
+    public function setOutput(OutputInterface $output)
+    {
+        $this->output = $output;
+        $this->output->setDecorated(true);
+    }
+
+    private function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
+    {
+        switch ($notification_code) {
+            case STREAM_NOTIFY_FILE_SIZE_IS:
+                $this->progressBar = new ProgressBar($this->output);
+                $this->progressBar->setBarWidth(100);
+                $this->progressBar->start($bytes_max);
+                break;
+            case STREAM_NOTIFY_PROGRESS:
+                $this->progressBar->setProgress($bytes_transferred);
+                break;
+            case STREAM_NOTIFY_COMPLETED:
+                $this->progressBar->setProgress($bytes_transferred);
+                $this->progressBar->finish();
+                break;
+        }
+    }
+
+    protected function downloadFile($url)
+    {
+        $temp = tempnam(sys_get_temp_dir(), 'printabrick.');
+
+        $ctx = stream_context_create([], [
+            'notification' => [$this, 'progressCallback'],
+        ]);
+
+        try {
+            if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) {
+                throw new LogicException('error writing file'); //TODO
+            }
+        } catch (ContextErrorException $e) {
+            throw new LogicException('wrong url'); //TODO
+        } catch (\Exception $e) {
+            throw new LogicException('exception:  '.$e->getMessage()); //TODO
+        }
+
+        return $temp;
+    }
+}

src/AppBundle/Command/Loader/RebrickableLoader.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace AppBundle\Loader;
+namespace AppBundle\Command\Loader;
 
 use AppBundle\Api\Manager\RebrickableManager;
 use AppBundle\Entity\BuildingKit;
@@ -9,7 +9,6 @@ use AppBundle\Entity\Color;
 use AppBundle\Entity\Keyword;
 use AppBundle\Entity\Part;
 use AppBundle\Entity\Part_BuildingKit;
-use Doctrine\ORM\EntityManager;
 use Symfony\Component\Console\Helper\ProgressBar;
 
 //TODO Refactor
@@ -20,34 +19,35 @@ class RebrickableLoader extends Loader
      */
     private $rebrickableManager;
 
+    private $rebrickable_url;
+
     /**
      * ModelLoaderService constructor.
      */
-    public function __construct($em, $rebrickableManager)
+    public function __construct($em, $rebrickableManager, $rebrickable_url)
     {
-        /*
-         * @var $em EntityManager
-         * */
         $this->em = $em;
         $this->rebrickableManager = $rebrickableManager;
+        $this->rebrickable_url = $rebrickable_url;
     }
 
     public function loadPartBuildingKits()
     {
+        $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com');
+        $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['set_pieces']);
+
         $partRepository = $this->em->getRepository('AppBundle:Part');
         $buldingKitRepository = $this->em->getRepository('AppBundle:BuildingKit');
         $colorRepository = $this->em->getRepository('AppBundle:Color');
 
-        $setPieces = tempnam(sys_get_temp_dir(), 'printabrick.');
-        file_put_contents($setPieces, fopen('compress.zlib://http://rebrickable.com/files/set_pieces.csv.gz', 'r'));
-
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
 
-        if (($handle = fopen($setPieces, 'r')) !== false) {
+        $this->output->writeln('Loading set_pieces.csv into Database');
+        if (($handle = fopen($file, 'r')) !== false) {
             $header = fgetcsv($handle, 200, ',');
 
             // create a new progress bar (50 units)
-            $progress = new ProgressBar($this->output, intval(exec("wc -l '$setPieces'"))); //TODO replace wc-l
+            $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l
             $progress->setFormat('very_verbose');
             $progress->setBarWidth(50);
             $progress->start();
@@ -83,26 +83,26 @@ class RebrickableLoader extends Loader
             $this->em->clear();
             fclose($handle);
             $progress->finish();
-            $progress->clear();
         }
 
-        unlink($setPieces);
+        unlink($file);
     }
 
     public function loadBuildingKits()
     {
-        $keywordRepository = $this->em->getRepository('AppBundle:Keyword');
+        $this->output->writeln('Downloading sets.csv from Rebrickable.com');
+        $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['sets']);
 
-        $sets = tempnam(sys_get_temp_dir(), 'printabrick.');
+        $keywordRepository = $this->em->getRepository('AppBundle:Keyword');
-        file_put_contents($sets, fopen('compress.zlib://http://rebrickable.com/files/sets.csv.gz', 'r'));
 
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
 
-        if (($handle = fopen($sets, 'r')) !== false) {
+        $this->output->writeln('Loading sets.csv into Database');
+        if (($handle = fopen($file, 'r')) !== false) {
             $header = fgetcsv($handle, 500, ',');
 
             // create a new progress bar (50 units)
-            $progress = new ProgressBar($this->output, intval(exec("wc -l '$sets'"))); //TODO replace wc-l
+            $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l
             $progress->setFormat('very_verbose');
             $progress->setBarWidth(50);
             $progress->start();
@@ -145,23 +145,23 @@ class RebrickableLoader extends Loader
             fclose($handle);
 
             $progress->finish();
-            $progress->clear();
         }
-        unlink($sets);
+        unlink($file);
     }
 
     public function loadParts()
     {
-        $pieces = tempnam(sys_get_temp_dir(), 'printabrick.');
+        $this->output->writeln('Downloading pieces.csv from Rebrickable.com');
-        file_put_contents($pieces, fopen('compress.zlib://http://rebrickable.com/files/pieces.csv.gz', 'r')); //TODO replace wc-l
+        $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['pieces']);
 
         $categoryRepository = $this->em->getRepository('AppBundle:Category');
 
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
 
-        if (($handle = fopen($pieces, 'r')) !== false) {
+        $this->output->writeln('Loading pieces.csv into Database');
+        if (($handle = fopen($file, 'r')) !== false) {
             // create a new progress bar (50 units)
-            $progress = new ProgressBar($this->output, intval(exec("wc -l '$pieces'")));
+            $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l
             $progress->setFormat('very_verbose');
             $progress->setBarWidth(50);
             $progress->start();
@@ -194,14 +194,15 @@ class RebrickableLoader extends Loader
             fclose($handle);
 
             $progress->finish();
-            $progress->clear();
         }
 
-        unlink($pieces);
+        unlink($file);
     }
 
     public function loadColors()
     {
+        $this->output->writeln('Loading colors into Database');
+
         $rb_colors = $this->rebrickableManager->getColors();
 
         foreach ($rb_colors as $rb_color) {
@@ -221,10 +222,10 @@ class RebrickableLoader extends Loader
     {
         $modelRepository = $this->em->getRepository('AppBundle:Model');
 
-        if (strpos($part->getNumber(), 'p')) {
+        $model = $modelRepository->findOneBy(['number' => $part->getNumber()]);
+
+        if (!$model && strpos($part->getNumber(), 'p')) {
             $model = $modelRepository->findOneBy(['number' => explode('p', $part->getNumber())[0]]);
-        } else {
-            $model = $modelRepository->findOneBy(['number' => $part->getNumber()]);
         }
 
         return $model;

src/AppBundle/Loader/Loader.php

@@ -1,22 +0,0 @@
-<?php
-
-namespace AppBundle\Loader;
-
-use Doctrine\ORM\EntityManager;
-use Symfony\Component\Console\Output\OutputInterface;
-
-class Loader
-{
-    /**
-     * @var EntityManager
-     */
-    protected $em;
-
-    protected $output;
-
-    public function setOutput(OutputInterface $output)
-    {
-        $this->output = $output;
-        $this->output->setDecorated(true);
-    }
-}