Add initial load command

app/config/service/loader.yml

@@ -3,11 +3,7 @@ services:
         abstract:  true
         class: AppBundle\Service\Loader\BaseLoader
         calls:
-          - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']]
+          - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader', '@app.transformer.format']]
-
-    service.ldview:
-        class: AppBundle\Service\LDViewService
-        arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']
 
     service.loader.rebrickable:
         class: AppBundle\Service\Loader\RebrickableLoader

app/config/service/service.yml

@@ -14,3 +14,7 @@ services:
     service.renderer.stl:
         class: AppBundle\Service\StlRendererService
         arguments: ['%kernel.root_dir%/Resources/povray_layout/layout.tmpl', '%povray_bin%','%stl2pov_bin%']
+
+    service.ldview:
+        class: AppBundle\Service\LDViewService
+        arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']

src/AppBundle/Command/InitDataCommand.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace AppBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\ArrayInput;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputDefinition;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class InitDataCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('app:init')
+            ->setDescription('Loads relations between LDraw models and Rebrickable parts.')
+            ->setHelp('This command allows you to load relation between models and parts into database.')
+            ->setDefinition(
+                new InputDefinition([
+                    new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
+                ])
+            );
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $loadModelsCommand = $this->getApplication()->find('app:load:models');
+
+        $returnCode = $loadModelsCommand->run(new ArrayInput([
+            'command' => 'app:load:models',
+            'ldraw' => $input->getArgument('ldraw'),
+            '--all' => true
+        ]),$output);
+
+        if($returnCode) {
+            return 1;
+        }
+
+        $loadRebrickableCommad = $this->getApplication()->find('app:load:rebrickable');
+        $returnCode = $loadRebrickableCommad->run(new ArrayInput([
+            'command' => 'app:load:rebrickable']),$output);
+
+        if($returnCode) {
+            return 1;
+        }
+
+        $loadRelationsCommand = $this->getApplication()->find('app:load:relations');
+
+        $returnCode = $loadRelationsCommand->run(new ArrayInput([
+            'command' => 'app:load:relations']),$output);
+
+        if($returnCode) {
+            return 1;
+        }
+
+        $loadImagesCommand = $this->getApplication()->find('app:load:images');
+
+        $returnCode = $loadImagesCommand->run(new ArrayInput([
+                'command' => 'app:load:images',
+                '--color' => -1,
+                '--rebrickable' => true,
+                '--models' => true
+            ]),$output);
+    }
+}

src/AppBundle/Command/LoadModelsCommand.php

@@ -23,7 +23,6 @@ class LoadModelsCommand extends ContainerAwareCommand
             ->setDefinition(
                 new InputDefinition([
                     new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
-//                    new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'),
                     new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'),
                     new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'),
                     new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'),
@@ -35,8 +34,7 @@ class LoadModelsCommand extends ContainerAwareCommand
     {
         if (!$this->lock()) {
             $output->writeln('The command is already running in another process.');
-
+            return 1;
-            return 0;
         }
 
         $modelLoader = $this->getContainer()->get('service.loader.model');
@@ -51,12 +49,10 @@ class LoadModelsCommand extends ContainerAwareCommand
             if (($path = $input->getOption('file')) != null) {
                 if ($file = realpath($path)) {
                     $output->writeln([
-                        'Loading model',
+                        "Loading model: {$path}",
-                        'path: '.$file,
-                        '------------------------------------------------------------------------------',
                     ]);
 
-                    $modelLoader->loadOneModel($file);
+                    $modelLoader->loadOne($file);
 
                     $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
                     $errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
@@ -70,10 +66,12 @@ class LoadModelsCommand extends ContainerAwareCommand
             // Load all models inside ldraw/parts directory
             if ($input->getOption('all')) {
                 $output->writeln([
-                    'Loading models from LDraw library: <comment>'.$ldrawPath.'</comment>',
+                    '<fg=cyan>------------------------------------------------------------------------------</>',
+                    "<fg=cyan>Loading models from LDraw library:</> <comment>{$ldrawPath}</comment>",
+                    '<fg=cyan>------------------------------------------------------------------------------</>',
                 ]);
 
-                $modelLoader->loadAllModels();
+                $modelLoader->loadAll();
 
                 $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
                 $errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
@@ -81,7 +79,9 @@ class LoadModelsCommand extends ContainerAwareCommand
                 $output->writeln(['Done with "'.$errors.'" errors.']);
             }
         } else {
-            $output->writeln($ldraw.' is not valid path');
+            $output->writeln("<error>{$ldraw} is not a valid path!</error>");
+            $this->release();
+            return 1;
         }
 
         $this->release();

src/AppBundle/Command/LoadRebrickableDataCommand.php

@@ -2,6 +2,7 @@
 
 namespace AppBundle\Command;
 
+use League\Flysystem\Exception;
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -21,7 +22,11 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
         $rebrickableLoader = $this->getContainer()->get('service.loader.rebrickable');
         $rebrickableLoader->setOutput($output);
 
-        //TODO log errors
+        try {
-        $rebrickableLoader->loadTables();
+            $rebrickableLoader->loadAll();
+        } catch (Exception $exception) {
+            $output->writeln("<error>{$exception->getMessage()}</error>");
+            return 1;
+        }
     }
 }

src/AppBundle/Command/LoadRelationCommand.php

@@ -21,7 +21,15 @@ class LoadRelationCommand extends ContainerAwareCommand
         $relationLoader = $this->getContainer()->get('service.loader.relation');
         $relationLoader->setOutput($output);
 
-        //TODO log errors
+
-        $relationLoader->loadNotPaired();
+        $output->writeln([
+            '<fg=cyan>------------------------------------------------------------------------------</>',
+            "<fg=cyan>Loading relations between parts and models...</>",
+            '<fg=cyan>------------------------------------------------------------------------------</>',
+        ]);
+
+        $relationLoader->loadAll();
+
+        $output->writeln(['<info>Done!</info>']);
     }
 }

src/AppBundle/Service/Loader/BaseLoader.php

@@ -4,6 +4,7 @@ namespace AppBundle\Service\Loader;
 
 use AppBundle\Exception\FileNotFoundException;
 use AppBundle\Exception\WriteErrorException;
+use AppBundle\Transformer\FormatTransformer;
 use Doctrine\ORM\EntityManager;
 use Monolog\Logger;
 use Symfony\Bundle\FrameworkBundle\Translation\Translator;
@@ -29,21 +30,29 @@ abstract class BaseLoader
      */
     protected $progressBar;
 
-    /** @var Logger */
+    /**
+     * @var Logger
+     */
     protected $logger;
 
+    /**
+     * @var FormatTransformer
+     */
+    private $formatTransformer;
+
     /**
      * Loader constructor.
      *
      * @param EntityManager $em
      * @param Translator    $translator
      */
-    public function setArguments(EntityManager $em, $logger)
+    public function setArguments(EntityManager $em, $logger, $formatTransformer)
     {
         $this->em = $em;
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
 
         $this->logger = $logger;
+        $this->formatTransformer = $formatTransformer;
     }
 
     public function setOutput(OutputInterface $output)
@@ -57,28 +66,33 @@ abstract class BaseLoader
      *
      * @param $total
      */
-    protected function initProgressBar($total)
+    protected function initProgressBar($total, $format = 'loader')
     {
         $this->progressBar = new ProgressBar($this->output, $total);
-//        $this->progressBar->setFormat('very_verbose');
+        ProgressBar::setFormatDefinition('loader', '[%current% / %max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%message%)' . PHP_EOL);
-        $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL);
+        ProgressBar::setFormatDefinition('download', '[%progress% / %size%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)' . PHP_EOL);
+        $this->progressBar->setFormat($format);
         $this->progressBar->setBarWidth(70);
+
         $this->progressBar->start();
     }
 
     protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
     {
+        $this->logger->info($notification_code);
+
         switch ($notification_code) {
             case STREAM_NOTIFY_FILE_SIZE_IS:
                 $this->initProgressBar($bytes_max);
-                $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL);
+                $this->progressBar->setFormat('download');
+                $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_max),'size');
+                $this->progressBar->setRedrawFrequency($bytes_max/20);
                 break;
             case STREAM_NOTIFY_PROGRESS:
                 $this->progressBar->setProgress($bytes_transferred);
+                $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_transferred),'progress');
                 break;
             case STREAM_NOTIFY_COMPLETED:
-                $this->progressBar->setMessage('<info>Done</info>');
-                $this->progressBar->setProgress($bytes_max);
                 $this->progressBar->finish();
                 break;
         }
@@ -106,6 +120,7 @@ abstract class BaseLoader
             if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) {
                 throw new WriteErrorException($temp);
             }
+            $this->progressBar->finish();
         } catch (ContextErrorException $e) {
             throw new FileNotFoundException($url);
         } catch (\Exception $e) {
@@ -114,4 +129,9 @@ abstract class BaseLoader
 
         return $temp;
     }
+
+    protected function writeOutput(array $lines) {
+        if($this->output)
+            $this->output->writeln($lines);
+    }
 }

src/AppBundle/Service/Loader/ModelLoader.php

@@ -12,15 +12,14 @@ use AppBundle\Exception\ConvertingFailedException;
 use AppBundle\Exception\FileException;
 use AppBundle\Exception\ParseErrorException;
 use AppBundle\Service\LDViewService;
-use AppBundle\Utils\LDModelParser;
+use AppBundle\Util\LDModelParser;
-use AppBundle\Utils\RelationMapper;
+use AppBundle\Util\RelationMapper;
 use League\Flysystem\Adapter\Local;
 use League\Flysystem\Exception;
 use League\Flysystem\Filesystem;
 use Symfony\Component\Finder\Finder;
 use Symfony\Component\Finder\SplFileInfo;
 
-//TODO refactor
 class ModelLoader extends BaseLoader
 {
     /**
@@ -80,7 +79,7 @@ class ModelLoader extends BaseLoader
         }
     }
 
-    public function loadOneModel($file)
+    public function loadOne($file)
     {
         $connection = $this->em->getConnection();
         try {
@@ -95,7 +94,7 @@ class ModelLoader extends BaseLoader
         }
     }
 
-    public function loadAllModels()
+    public function loadAll()
     {
         $files = $this->finder->in([
             $this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
@@ -109,7 +108,7 @@ class ModelLoader extends BaseLoader
             $connection->beginTransaction();
 
             try {
-                $this->progressBar->setMessage($file->getFilename(), 'filename');
+                $this->progressBar->setMessage($file->getFilename());
 
                 $this->loadModel($file->getRealPath());
 
@@ -127,7 +126,8 @@ class ModelLoader extends BaseLoader
     }
 
     /**
-     * Load Model entity into database.
+     * Load model entity and all related submodels into database while generating stl file of model.
+     * Uses LDView to convert LDraw .dat to .stl
      *
      * @param $file
      *
@@ -144,7 +144,8 @@ class ModelLoader extends BaseLoader
 
         // Parse model file save data to $modelArray
         try {
-            $modelArray = $this->ldModelParser->parse($file);
+            $content = file_get_contents($file);
+            $modelArray = $this->ldModelParser->parse($content);
         } catch (ParseErrorException $e) {
             $this->logger->error($e->getMessage(), [$file]);
 
@@ -172,8 +173,8 @@ class ModelLoader extends BaseLoader
 
                 return $parentModel;
             }
-            // Load model
 
+            // Load model
             $model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']);
 
             // Recursively load models of subparts
@@ -203,7 +204,7 @@ class ModelLoader extends BaseLoader
             try {
                 // update model only if newer version
                 if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) {
-                    $stl = $this->LDViewService->datToStl($file, $rewrite)->getPath();
+                    $stl = $this->LDViewService->datToStl($file, $this->rewite)->getPath();
                     $model->setPath($stl);
 
                     $model
@@ -271,13 +272,15 @@ class ModelLoader extends BaseLoader
             return $context->getAdapter()->getPathPrefix().$filename;
         }
         // Try to find model in current LDRAW\PARTS sub-directory
-        elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) {
+        elseif($this->ldrawLibraryContext) {
+            if($this->ldrawLibraryContext->has('parts/'.$filename)) {
                 return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
             }
             // Try to find model in current LDRAW\P sub-directory
             elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
                 return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
             }
+        }
 
         return null;
     }
@@ -297,6 +300,7 @@ class ModelLoader extends BaseLoader
             return new Filesystem($adapter);
         } catch (Exception $exception) {
             $this->logger->error($exception->getMessage());
+            return null;
         }
     }
 
@@ -322,7 +326,6 @@ class ModelLoader extends BaseLoader
         // Do not include models without permission to redistribute
         elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') {
             $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]);
-
             return false;
         }
 

src/AppBundle/Service/Loader/RebrickableLoader.php

@@ -21,7 +21,7 @@ class RebrickableLoader extends BaseLoader
         $this->rebrickable_url = $rebrickable_url;
     }
 
-    public function loadTables()
+    public function loadAll()
     {
         $connection = $this->em->getConnection();
         $connection->beginTransaction();
@@ -33,9 +33,8 @@ class RebrickableLoader extends BaseLoader
             $this->truncateTables();
             $connection->prepare('SET foreign_key_checks = 1;')->execute();
 
-            $this->output->writeln([
+            $this->writeOutput([
                 '<info>Truncated</info> <comment>rebrickable</comment> <info>database tables.</info>',
-                '------------------------------------------------------------------------------',
                 'Loading CSV files into database...',
             ]);
 
@@ -54,7 +53,7 @@ class RebrickableLoader extends BaseLoader
 
             $connection->commit();
 
-            $this->output->writeln('Rebrickable database loaded successfully!');
+            $this->writeOutput(['Rebrickable database loaded successfully!']);
         } catch (\Exception $e) {
             $connection->rollBack();
             throw $e;
@@ -65,19 +64,17 @@ class RebrickableLoader extends BaseLoader
     {
         $array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
 
-        $this->output->writeln([
+        $this->writeOutput([
-            'Loading Rebrickable CSV files',
+            '<fg=cyan>------------------------------------------------------------------------------</>',
-            '------------------------------------------------------------------------------',
+            '<fg=cyan>Loading Rebrickable CSV files</>',
+            '<fg=cyan>------------------------------------------------------------------------------</>',
         ]);
 
         foreach ($array as $item) {
             $this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
         }
 
-        $this->output->writeln([
+        $this->writeOutput(['<info>All CSV files loaded.</info>']);
-            '<info>Done</info>',
-            '------------------------------------------------------------------------------',
-        ]);
     }
 
     private function truncateTables()

src/AppBundle/Service/Loader/RelationLoader.php

@@ -30,26 +30,21 @@ class RelationLoader extends BaseLoader
     public function loadAll()
     {
         $parts = $this->em->getRepository(Part::class)->findAll();
-
+        $this->load($parts);
-        $this->initProgressBar(count($parts));
-        /** @var Part $part */
-        foreach ($parts as $part) {
-            $this->load($part);
-
-            $this->progressBar->advance();
-        }
-        $this->progressBar->finish();
     }
 
-    public function loadNotPaired()
+    public function loadNotPaired($parts)
     {
         $parts = $this->em->getRepository(Part::class)->findAllNotPaired();
+        $this->load($parts);
+    }
 
+    private function load($parts) {
         $this->initProgressBar(count($parts));
         /** @var Part $part */
         foreach ($parts as $part) {
-            $this->load($part);
+            $this->loadPartRelation($part);
-            $this->progressBar->setMessage($part->getNumber(), 'filename');
+            $this->progressBar->setMessage($part->getNumber());
             $this->progressBar->advance();
         }
         $this->progressBar->finish();
@@ -62,7 +57,7 @@ class RelationLoader extends BaseLoader
      *
      * @return Model $m
      */
-    private function load(Part $part)
+    private function loadPartRelation(Part $part)
     {
         $modelRepository = $this->em->getRepository(Model::class);