Improve load models command, log loading process

README.md

@@ -30,6 +30,7 @@ For full requirements see Symfony 3.2 [docs](http://symfony.com/doc/3.2/referenc
 #### Database
 1. Set application parameters in *app/config/parameters.yml*
 2. Generate empty database by running command `$ php bin/console doctrine:database:create`   
-3. Load LDraw models into database by running commad `$ php bin/console app:load:ldraw [ldraw_dir_path]` 
+3. Create database tables by running command `$ bin/console doctrine:schema:create`
+3. Load LDraw models into database by running commad `$ php bin/console app:load:models <ldraw_dir> [--all] [--file=FILE] [--update]`
 4. Load Rebrickable data into database by running command `$ php bin/console app:load:rebrickable`  
 5. Load relations between LDraw models and Rebrickable parts by running command `$ php bin/console app:load:relation`  

app/config/config_dev.yml

@@ -12,15 +12,22 @@ web_profiler:
     intercept_redirects: false
 
 monolog:
+    channels: ['loader']
     handlers:
         main:
             type: stream
             path: "%kernel.logs_dir%/%kernel.environment%.log"
             level: debug
-            channels: [!event]
+            channels: [!event, !loader]
         console:
             type:   console
             channels: [!event, !doctrine]
+        loader:
+            type: rotating_file
+            path: "%kernel.logs_dir%/loader.log"
+            level: debug
+            channels: 'loader'
+            max_files: 10
         # uncomment to get logging in your browser
         # you may have to allow bigger header sizes in your Web server configuration
         #firephp:

app/config/service/loader.yml

@@ -1,25 +1,25 @@
 services:
-    service.loader:
+    service.loader.base:
         abstract:  true
-        class: AppBundle\Service\Loader\BaseLoaderService
+        class: AppBundle\Service\Loader\BaseLoader
         calls:
-          - [setArguments, ['@doctrine.orm.entity_manager', '@app.relation.mapper']]
+          - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']]
 
     service.ldview:
         class: AppBundle\Service\LDViewService
         arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']
 
     service.loader.rebrickable:
-        class: AppBundle\Service\Loader\RebrickableLoaderService
+        class: AppBundle\Service\Loader\RebrickableLoader
-        arguments: ['%rebrickable_url%']
+        arguments: ['%rebrickable_csv_url%']
-        parent: service.loader
+        parent: service.loader.base
 
     service.loader.model:
-        class: AppBundle\Service\Loader\ModelLoaderService
+        class: AppBundle\Service\Loader\ModelLoader
-        arguments: ['@service.ldview', '@manager.ldraw', '@app.relation.mapper']
+        arguments: ['@service.ldview', '@app.relation.mapper']
-        parent: service.loader
+        parent: service.loader.base
 
     service.loader.relation:
         class: AppBundle\Service\Loader\RelationLoader
-        arguments: ['@manager.ldraw.model', '@repository.rebrickable.part', '@api.manager.rebrickable']
+        arguments: ['@api.manager.rebrickable', '@app.relation.mapper']
-        parent: service.loader
+        parent: service.loader.base

src/AppBundle/Command/LoadModelsCommand.php

@@ -19,18 +19,18 @@ class LoadModelsCommand extends ContainerAwareCommand
         $this
             ->setName('app:load:models')
             ->setDescription('Loads LDraw library models into database')
-            ->setHelp('This command allows you to load LDraw library models into while converting .dat files to .stl')
+            ->setHelp('This command allows you to load LDraw library models into database while converting .dat files to .stl format.')
             ->setDefinition(
                 new InputDefinition([
                     new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
-                    new InputOption('images', 'i',InputOption::VALUE_NONE, 'Do you want to generate images of models?'),
+//                    new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'),
-                    new InputOption('all','a',InputOption::VALUE_NONE, 'Do you want to load whole LDraw libary?'),
+                    new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'),
-                    new InputOption('file','f',InputOption::VALUE_REQUIRED, 'Path to DAT file that should be loaded into database')
+                    new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'),
+                    new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'),
                 ])
             );
     }
 
-    //TODO log errors
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         if (!$this->lock()) {
@@ -39,25 +39,49 @@ class LoadModelsCommand extends ContainerAwareCommand
             return 0;
         }
 
-        $ldrawLoader = $this->getContainer()->get('service.loader.model');
+        $modelLoader = $this->getContainer()->get('service.loader.model');
-        $ldrawLoader->setOutput($output);
+        $modelLoader->setOutput($output);
-        $ldrawLoader->setLDrawLibraryContext(realpath($input->getArgument('ldraw')));
+        $modelLoader->setRewite($input->getOption('update'));
 
-        try {
+        $ldraw = $input->getArgument('ldraw');
-            if (($ldrawPath = $input->getOption('file')) != null) {
-                $ldrawLoader->loadFileContext(dirname(realpath($ldrawPath)));
 
-                $model = $ldrawLoader->loadModel($ldrawPath);
+        if ($ldrawPath = realpath($ldraw)) {
+            $modelLoader->setLDrawLibraryContext($ldrawPath);
 
-                if($model !== null) {
+            if (($path = $input->getOption('file')) != null) {
-                    $this->getContainer()->get('manager.ldraw.model')->getRepository()->save($model);
+                if ($file = realpath($path)) {
+                    $output->writeln([
+                        'Loading model',
+                        'path: '.$file,
+                        '------------------------------------------------------------------------------',
+                    ]);
+
+                    $modelLoader->loadOneModel($file);
+
+                    $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
+                    $errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
+
+                    $output->writeln(['Done with "'.$errors.'" errors.']);
+                } else {
+                    $output->writeln("File $path not found");
                 }
             }
+
+            // Load all models inside ldraw/parts directory
             if ($input->getOption('all')) {
-               $ldrawLoader->loadAllModels();
+                $output->writeln([
+                    'Loading models from LDraw library: <comment>'.$ldrawPath.'</comment>',
+                ]);
+
+                $modelLoader->loadAllModels();
+
+                $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
+                $errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
+
+                $output->writeln(['Done with "'.$errors.'" errors.']);
             }
-        } catch (\Exception $e) {
+        } else {
-            printf($e->getMessage());
+            $output->writeln($ldraw.' is not valid path');
         }
 
         $this->release();

src/AppBundle/Exception/ErrorParsingLineException.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace AppBundle\Exception;
+
+use Throwable;
+
+class ErrorParsingLineException extends FileException
+{
+    public function __construct($path, $line, $message = '', $code = 0, Throwable $previous = null)
+    {
+        $message = sprintf('Error parsing line \""%s"\" "%s" file.',$line, $path);
+
+        parent::__construct($path, $message, $code, $previous);
+    }
+}

src/AppBundle/Exception/FileException.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace AppBundle\Exception;
+
+use Throwable;
+
+class FileException extends \Exception
+{
+    protected $path;
+
+    public function __construct($path, $message = '', $code = 0, Throwable $previous = null)
+    {
+        $this->path = $path;
+
+        parent::__construct($message, $code, $previous);
+    }
+}

src/AppBundle/Exception/FileNotFoundException.php

@@ -2,8 +2,14 @@
 
 namespace AppBundle\Exception;
 
+use Throwable;
 
-class FileNotFoundException extends \Symfony\Component\Filesystem\Exception\FileNotFoundException
+class FileNotFoundException extends FileException
 {
+    public function __construct($path, $message = '', $code = 0, Throwable $previous = null)
+    {
+        $message = sprintf('File "%s" not found.', $path);
 
+        parent::__construct($path, $message, $code, $previous);
+    }
 }

src/AppBundle/Exception/ParseErrorException.php

@@ -2,31 +2,14 @@
 
 namespace AppBundle\Exception;
 
+use Throwable;
 
-class ParseErrorException extends \Exception
+class ParseErrorException extends FileException
 {
-    private $filepath;
+    public function __construct($path, $message = '', $code = 0, Throwable $previous = null)
-
-    public function __construct($filepath = "", $message = "", $code = 0, Exception $previous = null)
     {
-        parent::__construct($message, $code, $previous);
+        $message = sprintf('Error parsing "%s" file.', $path);
 
-        $this->filepath = $filepath;
+        parent::__construct($path, $message, $code, $previous);
-    }
-
-    /**
-     * @return mixed
-     */
-    public function getFilepath()
-    {
-        return $this->filepath;
-    }
-
-    /**
-     * @param mixed $filepath
-     */
-    public function setFilepath($filepath)
-    {
-        $this->filepath = $filepath;
     }
 }

src/AppBundle/Exception/WriteErrorException.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace AppBundle\Exception;
+
+use Throwable;
+
+class WriteErrorException extends FileException
+{
+    public function __construct($file, $message = '', $code = 0, Throwable $previous = null)
+    {
+        $message = sprintf('Could not write into "%s" file.', $file);
+
+        parent::__construct($file, $message, $code, $previous);
+    }
+}

src/AppBundle/Service/LDViewService.php

@@ -3,11 +3,8 @@
 namespace AppBundle\Service;
 
 use AppBundle\Exception\ConvertingFailedException;
-use AppBundle\Exception\FileNotFoundException;
 use League\Flysystem\File;
 use League\Flysystem\Filesystem;
-use Symfony\Component\Asset\Exception\LogicException;
-use Symfony\Component\Finder\Finder;
 use Symfony\Component\Process\Exception\ProcessFailedException;
 use Symfony\Component\Process\ProcessBuilder;
 
@@ -27,15 +24,13 @@ class LDViewService
     /**
      * @var Filesystem
      */
-    private $ldrawLibraryFilesystem;
+    private $ldrawLibraryContext;
-
-    private $rewrite = false;
 
     /**
      * LDViewService constructor.
      *
-     * @param string     $ldview                    Path to LDView OSMesa binary file
+     * @param string     $ldview          Path to LDView OSMesa binary file
-     * @param Filesystem $mediaFilesystem           Filesystem for generated web assets
+     * @param Filesystem $mediaFilesystem Filesystem for generated web assets
      */
     public function __construct($ldview, $mediaFilesystem)
     {
@@ -44,11 +39,11 @@ class LDViewService
     }
 
     /**
-     * @param Filesystem $ldrawFilesystem
+     * @param string $ldrawLibraryContext
      */
-    public function setLdrawFilesystem($ldrawLibraryFilesystem)
+    public function setLDrawLibraryContext($ldrawLibraryContext)
     {
-        $this->ldrawLibraryFilesystem = $ldrawLibraryFilesystem;
+        $this->ldrawLibraryContext = $ldrawLibraryContext;
     }
 
     /**
@@ -57,35 +52,36 @@ class LDViewService
      *
      * @param $file
      *
-     * @return File
      * @throws ConvertingFailedException
+     *
+     * @return File
      */
-    public function datToStl($file)
+    public function datToStl($file, $rewrite = false)
     {
         if (!$this->mediaFilesystem->has('ldraw'.DIRECTORY_SEPARATOR.'models')) {
             $this->mediaFilesystem->createDir('ldraw'.DIRECTORY_SEPARATOR.'models');
         }
 
-        $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'models'.DIRECTORY_SEPARATOR.basename($file,'.dat').'.stl';
+        $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'models'.DIRECTORY_SEPARATOR.basename($file, '.dat').'.stl';
 
-        if (!file_exists($newFile) || $this->rewrite) {
+        if (!$this->mediaFilesystem->has($newFile) || $rewrite) {
             $this->runLDView([
                 $file,
-                '-LDrawDir='.$this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix(),
+                '-LDrawDir='.$this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
                 '-ExportFiles=1',
                 '-ExportSuffix=.stl',
                 '-ExportsDir='.$this->mediaFilesystem->getAdapter()->getPathPrefix().'ldraw'.DIRECTORY_SEPARATOR.'models',
             ]);
 
-
-
             // Check if file created successfully
-            if (!$this->mediaFilesystem->has($newFile)) {
+            if ($this->mediaFilesystem->has($newFile)) {
-                throw new ConvertingFailedException($newFile);
+                return $this->mediaFilesystem->get($newFile);
             }
+        } else {
+            return $this->mediaFilesystem->get($newFile);
         }
 
-        return $this->mediaFilesystem->get($newFile);
+        throw new ConvertingFailedException($file, 'STL');
     }
 
     /**
@@ -94,21 +90,22 @@ class LDViewService
      *
      * @param $file
      *
-     * @return File
      * @throws ConvertingFailedException
+     *
+     * @return File
      */
-    public function datToPng($file)
+    public function datToPng($file, $rewrite = false)
     {
         if (!$this->mediaFilesystem->has('ldraw'.DIRECTORY_SEPARATOR.'images')) {
             $this->mediaFilesystem->createDir('ldraw'.DIRECTORY_SEPARATOR.'images');
         }
 
-        $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'images'.DIRECTORY_SEPARATOR.basename($file,'.dat').'.png';
+        $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'images'.DIRECTORY_SEPARATOR.basename($file, '.dat').'.png';
 
         if (!$this->mediaFilesystem->has($newFile) || $this->rewrite) {
             $this->runLDView([
                 $file,
-                '-LDrawDir='.$this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix(),
+                '-LDrawDir='.$this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
                 '-AutoCrop=0',
                 '-SaveAlpha=0',
                 '-BackgroundColor3=0xFFFFFF',
@@ -126,12 +123,14 @@ class LDViewService
             ]);
 
             // Check if file created successfully
-            if (!$this->mediaFilesystem->has($newFile)) {
+            if ($this->mediaFilesystem->has($newFile)) {
-                throw new ConvertingFailedException($newFile);
+                return $this->mediaFilesystem->get($newFile);
             }
+        } else {
+            return $this->mediaFilesystem->get($newFile);
         }
 
-        return $this->mediaFilesystem->get($newFile);
+        throw new ConvertingFailedException($file, 'PNG');
     }
 
     /**

src/AppBundle/Service/Loader/BaseLoader.php

@@ -2,15 +2,17 @@
 
 namespace AppBundle\Service\Loader;
 
-use AppBundle\Utils\RelationMapper;
+use AppBundle\Exception\FileNotFoundException;
+use AppBundle\Exception\WriteErrorException;
 use Doctrine\ORM\EntityManager;
+use Monolog\Logger;
 use Symfony\Bundle\FrameworkBundle\Translation\Translator;
 use Symfony\Component\Asset\Exception\LogicException;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Debug\Exception\ContextErrorException;
 
-abstract class BaseLoaderService
+abstract class BaseLoader
 {
     /**
      * @var EntityManager
@@ -27,8 +29,8 @@ abstract class BaseLoaderService
      */
     protected $progressBar;
 
-    /** @var RelationMapper */
+    /** @var Logger */
-    protected $relationMapper;
+    protected $logger;
 
     /**
      * Loader constructor.
@@ -36,11 +38,12 @@ abstract class BaseLoaderService
      * @param EntityManager $em
      * @param Translator    $translator
      */
-    public function setArguments(EntityManager $em, $relationMapper)
+    public function setArguments(EntityManager $em, $logger)
     {
         $this->em = $em;
-        $this->relationMapper = $relationMapper;
         $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
+
+        $this->logger = $logger;
     }
 
     public function setOutput(OutputInterface $output)
@@ -49,11 +52,17 @@ abstract class BaseLoaderService
         $this->output->setDecorated(true);
     }
 
+    /**
+     * Initialize new progress bar.
+     *
+     * @param $total
+     */
     protected function initProgressBar($total)
     {
         $this->progressBar = new ProgressBar($this->output, $total);
-        $this->progressBar->setFormat('very_verbose');
+//        $this->progressBar->setFormat('very_verbose');
-        $this->progressBar->setFormat('%current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL);
+        $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL);
+        $this->progressBar->setBarWidth(70);
         $this->progressBar->start();
     }
 
@@ -62,20 +71,31 @@ abstract class BaseLoaderService
         switch ($notification_code) {
             case STREAM_NOTIFY_FILE_SIZE_IS:
                 $this->initProgressBar($bytes_max);
+                $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL);
                 break;
             case STREAM_NOTIFY_PROGRESS:
                 $this->progressBar->setProgress($bytes_transferred);
                 break;
             case STREAM_NOTIFY_COMPLETED:
-                $this->progressBar->setProgress($bytes_transferred);
+                $this->progressBar->setMessage('<info>Done</info>');
+                $this->progressBar->setProgress($bytes_max);
                 $this->progressBar->finish();
                 break;
         }
     }
 
+    /**
+     * Download file from $url, save it to system temp directory and return filepath.
+     *
+     * @param $url
+     *
+     * @throws FileNotFoundException
+     *
+     * @return bool|string
+     */
     protected function downloadFile($url)
     {
-        $this->output->writeln('Downloading file from: <info>'.$url.'</info>');
+        $this->output->writeln('Loading file from: <comment>'.$url.'</comment>');
         $temp = tempnam(sys_get_temp_dir(), 'printabrick.');
 
         $ctx = stream_context_create([], [
@@ -84,12 +104,12 @@ abstract class BaseLoaderService
 
         try {
             if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) {
-                throw new LogicException('error writing file'); //TODO
+                throw new WriteErrorException($temp);
             }
         } catch (ContextErrorException $e) {
-            throw new LogicException('wrong url'); //TODO
+            throw new FileNotFoundException($url);
         } catch (\Exception $e) {
-            throw new LogicException('exception:  '.$e->getMessage()); //TODO
+            throw new LogicException($e);
         }
 
         return $temp;

src/AppBundle/Service/Loader/ModelLoader.php

@@ -0,0 +1,331 @@
+<?php
+
+namespace AppBundle\Service\Loader;
+
+use AppBundle\Entity\LDraw\Alias;
+use AppBundle\Entity\LDraw\Author;
+use AppBundle\Entity\LDraw\Category;
+use AppBundle\Entity\LDraw\Keyword;
+use AppBundle\Entity\LDraw\Model;
+use AppBundle\Entity\LDraw\Subpart;
+use AppBundle\Exception\ConvertingFailedException;
+use AppBundle\Exception\FileException;
+use AppBundle\Exception\ParseErrorException;
+use AppBundle\Service\LDViewService;
+use AppBundle\Utils\LDModelParser;
+use AppBundle\Utils\RelationMapper;
+use League\Flysystem\Adapter\Local;
+use League\Flysystem\Exception;
+use League\Flysystem\Filesystem;
+use Symfony\Component\Finder\Finder;
+use Symfony\Component\Finder\SplFileInfo;
+
+//TODO refactor
+class ModelLoader extends BaseLoader
+{
+    /**
+     * @var Filesystem
+     */
+    private $ldrawLibraryContext;
+
+    /**
+     * @var LDViewService
+     */
+    private $LDViewService;
+
+    /** @var LDModelParser */
+    private $ldModelParser;
+
+    /** @var RelationMapper */
+    private $relationMapper;
+
+    /** @var Finder */
+    private $finder;
+
+    private $rewite = false;
+
+    /**
+     * LDrawLoaderService constructor.
+     *
+     * @param LDViewService  $LDViewService
+     * @param RelationMapper $relationMapper
+     */
+    public function __construct($LDViewService, $relationMapper)
+    {
+        $this->LDViewService = $LDViewService;
+        $this->relationMapper = $relationMapper;
+        $this->ldModelParser = new LDModelParser();
+        $this->finder = new Finder();
+    }
+
+    /**
+     * @param bool $rewite
+     */
+    public function setRewite($rewite)
+    {
+        $this->rewite = $rewite;
+    }
+
+    /**
+     * @param $ldrawLibrary
+     */
+    public function setLDrawLibraryContext($ldrawLibrary)
+    {
+        try {
+            $adapter = new Local($ldrawLibrary);
+            $this->ldrawLibraryContext = new Filesystem($adapter);
+            $this->LDViewService->setLDrawLibraryContext($this->ldrawLibraryContext);
+        } catch (Exception $exception) {
+            $this->logger->error($exception->getMessage());
+        }
+    }
+
+    public function loadOneModel($file)
+    {
+        $connection = $this->em->getConnection();
+        try {
+            $connection->beginTransaction();
+
+            $this->loadModel($file);
+
+            $connection->commit();
+        } catch (\Exception $e) {
+            $connection->rollBack();
+            $this->logger->error($e->getMessage());
+        }
+    }
+
+    public function loadAllModels()
+    {
+        $files = $this->finder->in([
+            $this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
+        ])->path('parts/')->name('*.dat')->depth(1)->files();
+
+        $this->initProgressBar($files->count());
+
+        /** @var SplFileInfo $file */
+        foreach ($files as $file) {
+            $connection = $this->em->getConnection();
+            $connection->beginTransaction();
+
+            try {
+                $this->progressBar->setMessage($file->getFilename(), 'filename');
+
+                $this->loadModel($file->getRealPath());
+
+                $connection->commit();
+            } catch (\Exception $exception) {
+                $connection->rollBack();
+                $this->logger->error($exception->getMessage());
+            }
+
+            $connection->close();
+
+            $this->progressBar->advance();
+        }
+        $this->progressBar->finish();
+    }
+
+    /**
+     * Load Model entity into database.
+     *
+     * @param $file
+     *
+     * @return Model|null|false
+     */
+    public function loadModel($file)
+    {
+        $fileContext = $this->getFileContext($file);
+
+        // Return model from database if rewrite is not enabled
+        if (!$this->rewite && $model = $this->em->getRepository(Model::class)->findOneByNumber(basename($file, '.dat'))) {
+            return $model;
+        }
+
+        // Parse model file save data to $modelArray
+        try {
+            $modelArray = $this->ldModelParser->parse($file);
+        } catch (ParseErrorException $e) {
+            $this->logger->error($e->getMessage(), [$file]);
+
+            return null;
+        } catch (FileException $e) {
+            $this->logger->error($e->getMessage(), [$file]);
+
+            return null;
+        }
+
+        // Check if model fulfills rules and should be loaded
+        if ($this->isModelIncluded($modelArray)) {
+            // Recursively load model parent (if any) and add model id as alias of parent
+            if (($parentId = $this->getParentId($modelArray)) && ($parentModelFile = $this->findSubmodelFile($parentId, $fileContext)) !== null) {
+                $parentModel = $this->loadModel($parentModelFile);
+
+                if ($parentModel) {
+                    $alias = $this->em->getRepository(Alias::class)->getOrCreate($modelArray['id'], $parentModel);
+                    $parentModel->addAlias($alias);
+
+                    $this->em->getRepository(Model::class)->save($parentModel);
+                } else {
+                    $this->logger->info('Model skipped. ', ['number' => $modelArray['id'], 'parent' => $modelArray['parent']]);
+                }
+
+                return $parentModel;
+            }
+            // Load model
+
+            $model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']);
+
+            // Recursively load models of subparts
+            if (isset($modelArray['subparts'])) {
+                foreach ($modelArray['subparts'] as $subpartId => $count) {
+                    // Try to find model of subpart
+                    if (($subpartFile = $this->findSubmodelFile($subpartId, $fileContext)) !== null) {
+                        $subModel = $this->loadModel($subpartFile);
+                        if ($subModel) {
+                            $subpart = $this->em->getRepository(Subpart::class)->getOrCreate($model, $subModel, $count);
+                            $model->addSubpart($subpart);
+                        }
+                    } else {
+                        $this->logger->error('Subpart file not found', ['subpart' => $subpartId, 'model' => $modelArray]);
+                    }
+                }
+            }
+
+            // Add Keywords to model
+            if (isset($modelArray['keywords'])) {
+                foreach ($modelArray['keywords'] as $keyword) {
+                    $keyword = $this->em->getRepository(Keyword::class)->getOrCreate(stripslashes(strtolower(trim($keyword))));
+                    $model->addKeyword($keyword);
+                }
+            }
+
+            try {
+                // update model only if newer version
+                if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) {
+                    $stl = $this->LDViewService->datToStl($file, $rewrite)->getPath();
+                    $model->setPath($stl);
+
+                    $model
+                        ->setName($modelArray['name'])
+                        ->setCategory($this->em->getRepository(Category::class)->getOrCreate($modelArray['category']))
+                        ->setAuthor($this->em->getRepository(Author::class)->getOrCreate($modelArray['author']))
+                        ->setModified($modelArray['modified']);
+
+                    $this->em->getRepository(Model::class)->save($model);
+                }
+            } catch (ConvertingFailedException $e) {
+                $this->logger->error($e->getMessage());
+
+                return null;
+            }
+
+            return $model;
+        }
+
+        return false;
+    }
+
+    /**
+     * Get parent id of model from alias_model.yml if defined or return parent id loaded by LDModelParser.
+     *
+     * Used to eliminate duplicites of models in library, that could not be determined automatically
+     *
+     * @param $modelArray
+     *
+     * @return string
+     */
+    private function getParentId($modelArray)
+    {
+        if ($this->relationMapper->find($modelArray['id'], 'alias_model') !== $modelArray['id']) {
+            return $this->relationMapper->find($modelArray['id'], 'alias_model');
+        }
+
+        return $modelArray['parent'];
+    }
+
+    /**
+     * Find submodel file inside model context or inside LDraw library.
+     *
+     *
+     * LDraw.org Standards: File Format 1.0.2 (http://www.ldraw.org/article/218.html)
+     *
+     *  "Sub-files can be located in the LDRAW\PARTS sub-directory, the LDRAW\P sub-directory, the LDRAW\MODELS sub-directory,
+     *  the current file's directory, a path relative to one of these directories, or a full path may be specified. Sub-parts are typically
+     *  stored in the LDRAW\PARTS\S sub-directory and so are referenced as s\subpart.dat, while hi-res primitives are stored in the
+     *  LDRAW\P\48 sub-directory and so referenced as 48\hires.dat"
+     *
+     *
+     * @param $id
+     * @param Filesystem $context
+     *
+     * @return string
+     */
+    private function findSubmodelFile($id, $context)
+    {
+        // Replace "\" directory separator used inside ldraw model files with system directoru separator
+        $filename = str_replace('\\', DIRECTORY_SEPARATOR, strtolower($id).'.dat');
+
+        // Try to find model in current file's directory
+        if ($context->has($filename)) {
+            return $context->getAdapter()->getPathPrefix().$filename;
+        }
+        // Try to find model in current LDRAW\PARTS sub-directory
+        elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) {
+            return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
+        }
+        // Try to find model in current LDRAW\P sub-directory
+        elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
+            return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
+        }
+
+        return null;
+    }
+
+    /**
+     * Get new filesystem context of current file.
+     *
+     * @param $file
+     *
+     * @return Filesystem
+     */
+    private function getFileContext($file)
+    {
+        try {
+            $adapter = new Local(dirname($file));
+
+            return new Filesystem($adapter);
+        } catch (Exception $exception) {
+            $this->logger->error($exception->getMessage());
+        }
+    }
+
+    /**
+     * Determine if model file should be loaded into database.
+     *
+     * @param $modelArray
+     *
+     * @return bool
+     */
+    private function isModelIncluded($modelArray)
+    {
+        // Do not include part primitives and subparts
+        if (in_array($modelArray['type'], ['48_Primitive', '8_Primitive', 'Primitive', 'Subpart'])) {
+            return false;
+        }
+        // Do not include sticker models
+        elseif ($modelArray['type'] == 'Sticker') {
+            $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'type' => $modelArray['type']]);
+
+            return false;
+        }
+        // Do not include models without permission to redistribute
+        elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') {
+            $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]);
+
+            return false;
+        }
+
+        return true;
+    }
+}

src/AppBundle/Service/Loader/ModelLoaderService.php

@@ -1,250 +0,0 @@
-<?php
-
-namespace AppBundle\Service\Loader;
-
-use AppBundle\Entity\LDraw\Alias;
-use AppBundle\Entity\LDraw\Model;
-use AppBundle\Entity\LDraw\Type;
-use AppBundle\Exception\ConvertingFailedException;
-use AppBundle\Manager\LDrawManager;
-use AppBundle\Service\LDViewService;
-use AppBundle\Utils\DatParser;
-use AppBundle\Utils\RelationMapper;
-use League\Flysystem\Adapter\Local;
-use League\Flysystem\Filesystem;
-use Symfony\Component\Finder\Finder;
-use Symfony\Component\Finder\SplFileInfo;
-
-//TODO refactor
-class ModelLoaderService extends BaseLoaderService
-{
-    /**
-     * @var Filesystem
-     */
-    private $ldrawLibraryContext;
-
-    /**
-     * @var Filesystem
-     */
-    private $fileContext;
-
-    /**
-     * @var LDViewService
-     */
-    private $LDViewService;
-
-    /** @var LDrawManager */
-    private $ldrawService;
-
-    /** @var DatParser */
-    private $datParser;
-
-    private $newModels;
-
-    /** @var RelationMapper */
-    protected $relationMapper;
-
-    /** @var Finder */
-    private $finder;
-
-    /**
-     * LDrawLoaderService constructor.
-     * @param LDViewService $LDViewService
-     * @param LDrawManager $ldrawService
-     * @param RelationMapper $relationMapper
-     */
-    public function __construct($LDViewService, $ldrawService, $relationMapper)
-    {
-        $this->LDViewService = $LDViewService;
-        $this->ldrawService = $ldrawService;
-        $this->relationMapper = $relationMapper;
-
-        $this->datParser = new DatParser();
-        $this->finder = new Finder();
-    }
-
-    public function setLDrawLibraryContext($ldrawLibrary)
-    {
-        $adapter = new Local($ldrawLibrary);
-        $this->ldrawLibraryContext = new Filesystem($adapter);
-
-        $this->LDViewService->setLdrawFilesystem($this->ldrawLibraryContext);
-    }
-
-    public function setFileContext($file) {
-        $adapter = new Local($file);
-        $this->fileContext = new Filesystem($adapter);
-    }
-
-    public function loadAllModels()
-    {
-        $files = $this->finder->in([$this->ldrawLibraryContext->getAdapter()->getPathPrefix()])->path('parts/')->name('*.dat')->depth(1)->files();
-
-        $modelManager = $this->ldrawService->getModelManager();
-
-        $this->initProgressBar($files->count());
-
-        /** @var SplFileInfo $file */
-        foreach ($files as $file) {
-            $this->newModels = [];
-
-            $model = $this->loadModel($file->getRealPath());
-
-            if($model !== null) {
-                $modelManager->getRepository()->save($model);
-            }
-
-            $this->progressBar->advance();
-        }
-        $this->progressBar->finish();
-    }
-
-    /**
-     * Load Model entity into database.
-     *
-     * @param $file
-     *
-     * @return Model|null
-     */
-    public function loadModel($file)
-    {
-        $model = null;
-
-        $modelManager = $this->ldrawService->getModelManager();
-        $subpartManager = $this->ldrawService->getSubpartManager();
-
-        if(($model = $modelManager->findByNumber(basename($file,'.dat'))) || ($model = isset($this->newModels[basename($file,'.dat')]) ? $this->newModels[basename($file,'.dat')] : null)) {
-            return $model;
-        }
-
-        try {
-            $modelArray = $this->datParser->parse($file);
-        } catch (\Exception $e) {
-            dump($e->getMessage());
-            return null;
-        }
-
-        if ($this->isModelIncluded($modelArray)) {
-
-            if ($parentModelFile = $this->getParentModelFile($modelArray)) {
-                try {
-                    if(($parentModel = $this->loadModel($parentModelFile))!= null) {
-                        $alias = $this->ldrawService->getAliasManager()->create($modelArray['id'], $parentModel);
-                        $parentModel->addAlias($alias);
-
-                        $this->newModels[$parentModel->getNumber()] = $parentModel;
-                        return $parentModel;
-                    }
-                } catch (\Exception $e) {
-                    dump('b');
-                    dump($e->getMessage());
-                    return null;
-                }
-            } else {
-                $model = $modelManager->create($modelArray['id']);
-
-                if (isset($modelArray['keywords'])) {
-                    foreach ($modelArray['keywords'] as $keyword) {
-                        $keyword = stripslashes(strtolower(trim($keyword)));
-                        $model->addKeyword($this->ldrawService->getKeywordManager()->create($keyword));
-                    }
-                }
-
-                if (isset($modelArray['subparts'])) {
-                    foreach ($modelArray['subparts'] as $subpartId => $count) {
-                        if(strpos($subpartId, 's\\') === false) {
-                            if(($subpartFile = $this->findModelFile($subpartId)) != null) {
-                                try {
-                                    if ($subModel = $this->loadModel($subpartFile)) {
-                                        $subpart = $subpartManager->create($model,$subModel,$count);
-
-                                        $model->addSubpart($subpart);
-                                    }
-                                } catch (\Exception $e) {
-                                    dump('c');
-                                    dump($e->getMessage());
-                                }
-                            }
-                        }
-                    }
-                }
-
-                $model
-                    ->setName($modelArray['name'])
-                    ->setCategory($this->ldrawService->getCategoryManager()->create($modelArray['category']))
-                    ->setAuthor($modelArray['author'])
-                    ->setModified($modelArray['modified'])
-                    ->setPath($this->loadStlModel($file));
-
-                $this->LDViewService->datToPng($file);
-
-                $this->newModels[$model->getNumber()] = $model;
-            }
-        }
-
-        return $model;
-    }
-
-    private function getParentModelFile($modelArray) {
-        if($this->relationMapper->find($modelArray['id'], 'alias_model') !== $modelArray['id']) {
-            return $this->findModelFile($this->relationMapper->find($modelArray['id'], 'alias_model'));
-        } else {
-            return strpos($modelArray['parent'], 's\\') === false ? $this->findModelFile($modelArray['parent']) : null;
-        }
-    }
-
-    /**
-     * Find model file on ldraw filesystem.
-     *
-     * @param $id
-     *
-     * @return string
-     */
-    private function findModelFile($id)
-    {
-        $filename = strtolower($id).'.dat';
-
-        if($this->fileContext && $this->fileContext->has($filename)) {
-            return $this->fileContext->getAdapter()->getPathPrefix().$filename;
-        } else if ($this->ldrawLibraryContext->has('parts/'.$filename)) {
-            return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts/'.$filename;
-        }
-
-        return null;
-    }
-
-    /**
-     * Determine if model file should be loaded into database.
-     *
-     * @param $modelArray
-     *
-     * @return bool
-     */
-    private function isModelIncluded($modelArray)
-    {
-        // Do not include sticker parts and incomplete parts
-        if ( $modelArray['type'] != 'Subpart' && $modelArray['type'] != 'Sticker' ) {
-            return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * Load stl model by calling LDViewSevice and create new Model.
-     *
-     * @param $file
-     *
-     * @throws \Exception
-     *
-     * @return string path of stl file
-     */
-    private function loadStlModel($file)
-    {
-        try {
-            return $this->LDViewService->datToStl($file)->getPath();
-        } catch (ConvertingFailedException $e) {
-            throw $e; //TODO
-        }
-    }
-}

src/AppBundle/Utils/LDModelParser.php

@@ -5,13 +5,11 @@ namespace AppBundle\Utils;
 use AppBundle\Exception\FileNotFoundException;
 use AppBundle\Exception\ParseErrorException;
 use League\Flysystem\File;
-use Symfony\Component\Asset\Exception\LogicException;
 
-class DatParser
+class LDModelParser
 {
     /**
-     * Parse LDraw .dat file header identifying model store data to array.
+     * Parse LDraw  model .dat file and return associative array in format:
-     *
      * [
      *  'id' => string
      *  'name' => string
@@ -20,17 +18,19 @@ class DatParser
      *  'author' => string
      *  'modified' => DateTime
      *  'type' => string
-     *  'subparts' => []
+     *  'subparts' => [],
-     * ]
+     *  'licence' => string
+     * ].
      *
      * LDraw.org Standards: Official Library Header Specification (http://www.ldraw.org/article/398.html)
      *
-     * @return array
      * @throws FileNotFoundException|ParseErrorException
+     *
+     * @return array
      */
     public function parse($file)
     {
-        if(file_exists($file)) {
+        if (file_exists($file)) {
             $model = [
                 'id' => null,
                 'name' => null,
@@ -40,7 +40,8 @@ class DatParser
                 'modified' => null,
                 'type' => null,
                 'subparts' => [],
-                'parent' => null
+                'parent' => null,
+                'license' => null,
             ];
 
             try {
@@ -63,19 +64,24 @@ class DatParser
                                 $model['name'] = preg_replace('/ {2,}/', ' ', ltrim($line, '=_'));
 
                                 $firstLine = true;
-                            } // 0 !CATEGORY <CategoryName>
+                            }
+                            // 0 !CATEGORY <CategoryName>
                             elseif (strpos($line, '!CATEGORY ') === 0) {
                                 $model['category'] = trim(preg_replace('/^!CATEGORY /', '', $line));
-                            } // 0 !KEYWORDS <first keyword>, <second keyword>, ..., <last keyword>
+                            }
+                            // 0 !KEYWORDS <first keyword>, <second keyword>, ..., <last keyword>
                             elseif (strpos($line, '!KEYWORDS ') === 0) {
                                 $model['keywords'] = explode(', ', preg_replace('/^!KEYWORDS /', '', $line));
-                            } // 0 Name: <Filename>.dat
+                            }
+                            // 0 Name: <Filename>.dat
                             elseif (strpos($line, 'Name: ') === 0 && !isset($header['id'])) {
                                 $model['id'] = preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line);
-                            } // 0 Author: <Realname> [<Username>]
+                            }
+                            // 0 Author: <Realname> [<Username>]
                             elseif (strpos($line, 'Author: ') === 0) {
                                 $model['author'] = preg_replace('/^Author: /', '', $line);
-                            } // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR
+                            }
+                            // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR
                             elseif (strpos($line, '!LDRAW_ORG ') === 0) {
                                 $type = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE| ORIGINAL)(.*)/', '$2', $line);
 
@@ -84,17 +90,23 @@ class DatParser
                                 // Last modification date in format YYYY-RR
                                 $date = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE | ORIGINAL )(.*)/', '$4', $line);
                                 if (preg_match('/^[1-2][0-9]{3}-[0-9]{2}$/', $date)) {
-                                    $model['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date . '-01 00:00:00');
+                                    $model['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date.'-01 00:00:00');
                                 }
                             }
+                            // 0 !LICENSE Redistributable under CCAL version 2.0 : see CAreadme.txt | 0 !LICENSE Not redistributable : see NonCAreadme.txt
+                            elseif (strpos($line, '!LICENSE ') === 0) {
+                                $model['license'] = preg_replace('/(^!LICENSE )(.*) : (.*)$/', '$2', $line);
+                            }
                         } elseif (strpos($line, '1 ') === 0) {
-                            $id = $this->getReferencedModelNumber($line);
+                            $id = strtolower($this->getReferencedModelNumber($line));
 
-                            if(isset($model['subparts'][$id])) {
+                            if (isset($model['subparts'][$id])) {
                                 $model['subparts'][$id] = $model['subparts'][$id] + 1;
                             } else {
                                 $model['subparts'][$id] = 1;
                             }
+                        } elseif (!empty($line) && !in_array($line[0], ['2', '3', '4', '5'])) {
+                            throw new ErrorParsingLineException($file,$line);
                         }
                     }
 
@@ -102,14 +114,12 @@ class DatParser
                         $model['type'] = 'Sticker';
                     } elseif (count($model['subparts']) == 1 && in_array($model['type'], ['Part Alias', 'Shortcut Physical_Colour', 'Shortcut Alias', 'Part Physical_Colour'])) {
                         $model['parent'] = array_keys($model['subparts'])[0];
-                    } elseif ($parent = $this->getPrintedModelParentNumber($model['id'])) {
+                    } elseif (($parent = $this->getPrintedModelParentNumber($model['id'])) && !in_array($model['type'], ['48_Primitive', '8_Primitive', 'Primitive', 'Subpart'])) {
                         $model['type'] = 'Printed';
                         $model['parent'] = $parent;
                     } elseif ($parent = $this->getObsoleteModelParentNumber($model['name'])) {
                         $model['type'] = 'Alias';
                         $model['parent'] = $parent;
-                    } elseif (strpos($model['name'], '~') === 0 && $model['type'] != 'Alias') {
-                        $model['type'] = 'Obsolete/Subpart';
                     }
 
                     fclose($handle);
@@ -137,10 +147,10 @@ class DatParser
      */
     public function getReferencedModelNumber($line)
     {
-        if(preg_match('/^1 16 0 0 0 -1 0 0 0 1 0 0 0 1 (.*)\.(dat|DAT)$/', $line, $matches))
+        $line = ($line);
-            return null;
+
-        if (preg_match('/^1(.*) (.*)\.(dat|DAT)$/', $line, $matches)) {
+        if (preg_match('/^1(.*) (.*)\.dat$/', strtolower($line), $matches)) {
-            return $matches[2];
+            return str_replace('\\', DIRECTORY_SEPARATOR, $matches[2]);
         }
 
         return null;
@@ -160,7 +170,7 @@ class DatParser
      */
     public function getPrintedModelParentNumber($id)
     {
-        if (preg_match('/(^.*)(p[0-9a-z][0-9a-z][0-9a-z]{0,1})$/', $id, $matches)) {
+        if (preg_match('/(^.*)(p[0-9a-z]{2,3})$/', $id, $matches)) {
             return $matches[1];
         }
 
@@ -178,7 +188,7 @@ class DatParser
      * @param $name
      * @param $number
      *
-     * @return string|null LDraw number of printed part parent
+     * @return bool
      */
     public function isSticker($name, $number)
     {
@@ -186,8 +196,8 @@ class DatParser
             return true;
         }
 
-        // Check if in format nnnDaa == sticker
+        // Check if in format n*Daa == sticker
-        return preg_match('/(^.*)(d[a-z0-9][a-z0-9])$/', $number);
+        return preg_match('/(^.*)(d[0-9]{2})$/', $number);
     }
 
     /**
@@ -205,7 +215,7 @@ class DatParser
     public function getObsoleteModelParentNumber($name)
     {
         if (preg_match('/^(~Moved to )(.*)$/', $name, $matches)) {
-            return $matches[2];
+            return str_replace('\\', DIRECTORY_SEPARATOR, $matches[2]);
         }
 
         return null;