Recursive load ldraw model subparts, Enable single model load

app/config/service/loader.yml

@@ -14,13 +14,9 @@ services:
         arguments: ['%rebrickable_url%']
         parent: service.loader
 
-    util.dat.parser:
-        class: AppBundle\Utils\DatParser
-        arguments: ['@app.relation.mapper']
-
     service.loader.ldraw:
         class: AppBundle\Service\Loader\LDrawLoaderService
-        arguments: ['@oneup_flysystem.ldraw_library_filesystem','@service.ldview', '%ldraw_url%', '@manager.ldraw', '@util.dat.parser']
+        arguments: ['@oneup_flysystem.ldraw_library_filesystem','@service.ldview', '@manager.ldraw', '@app.relation.mapper']
         parent: service.loader
 
     service.loader.relation:

src/AppBundle/Command/LoadLDRawLibraryCommand.php

@@ -1,39 +0,0 @@
-<?php
-
-namespace AppBundle\Command;
-
-use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
-use Symfony\Component\Console\Input\InputArgument;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Output\OutputInterface;
-
-class LoadLDRawLibraryCommand extends ContainerAwareCommand
-{
-    protected function configure()
-    {
-        $this
-            ->setName('app:load:ldraw')
-            ->setDescription('Loads LDraw library parts')
-            ->setHelp('This command allows you to..')
-            ->addArgument('file', InputArgument::OPTIONAL, 'Model to load');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {
-        $ldrawLoader = $this->getContainer()->get('service.loader.ldraw');
-        $ldrawLoader->setOutput($output);
-
-        //TODO log errors
-
-        try {
-            // TODO handle relative path to dir
-            if (($ldrawPath = $input->getArgument('file')) != null) {
-                $ldrawPath = $ldrawLoader->loadModel($ldrawPath);
-            } else {
-                $ldrawLoader->loadAllModels();
-            }
-        } catch (\Exception $e) {
-            printf($e->getMessage());
-        }
-    }
-}

src/AppBundle/Command/LoadModelsCommand.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace AppBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputDefinition;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class LoadModelsCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('app:load:models')
+            ->setDescription('Loads LDraw library models into database')
+            ->setHelp('This command allows you to load LDraw library models into while converting .dat files to .stl')
+            ->setDefinition(
+                new InputDefinition(array(
+                    new InputOption('images', 'i'),
+                    new InputOption('ldraw', 'l', InputOption::VALUE_REQUIRED),
+                    new InputOption('file', 'f', InputOption::VALUE_REQUIRED),
+                ))
+            );
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $ldrawLoader = $this->getContainer()->get('service.loader.ldraw');
+        $ldrawLoader->setOutput($output);
+
+        //TODO log errors
+        try {
+            // TODO handle relative path to dir
+            if (($ldrawPath = $input->getOption('file')) != null) {
+                $ldrawLoader->loadFileContext($ldrawPath);
+
+                $model = $ldrawLoader->loadModel($ldrawPath);
+
+                if($model !== null) {
+                    $this->getContainer()->get('manager.ldraw.model')->getRepository()->save($model);
+                }
+            } else {
+                $ldrawLoader->loadAllModels();
+            }
+        } catch (\Exception $e) {
+            printf($e->getMessage());
+        }
+    }
+}

src/AppBundle/Manager/LDraw/SubpartManager.php

@@ -25,17 +25,17 @@ class SubpartManager extends BaseManager
      *
      * @return Subpart
      */
-    public function create($parent, $child)
+    public function create($parent, $child, $count)
     {
-        if (($subpart = $this->repository->findOneByKeys($parent, $child))) {
-            $subpart->setCount($subpart->getCount() + 1);
-        } else {
+//        if (($subpart = $this->repository->findOneByKeys($parent, $child))) {
+//            $subpart->setCount($count);
+//        } else {
             $subpart = new Subpart();
             $subpart
                 ->setParent($parent)
                 ->setSubpart($child)
-                ->setCount(1);
-        }
+                ->setCount($count);
+//        }
 
         return $subpart;
     }

src/AppBundle/Service/Loader/LDrawLoaderService.php

@@ -2,19 +2,17 @@
 
 namespace AppBundle\Service\Loader;
 
-use AppBundle\Entity\LDraw\Category;
+use AppBundle\Entity\LDraw\Alias;
 use AppBundle\Entity\LDraw\Model;
 use AppBundle\Entity\LDraw\Type;
-use AppBundle\Exception\FileNotFoundException;
 use AppBundle\Manager\LDrawManager;
 use AppBundle\Service\LDViewService;
 use AppBundle\Utils\DatParser;
+use AppBundle\Utils\RelationMapper;
 use League\Flysystem\Adapter\Local;
-use League\Flysystem\File;
 use League\Flysystem\Filesystem;
+
 //use Symfony\Component\Asset\Exception\LogicException;
-use Symfony\Component\Asset\Exception\LogicException;
-use Symfony\Component\Console\Helper\ProgressBar;
 
 //TODO refactor
 class LDrawLoaderService extends BaseLoaderService
@@ -22,12 +20,12 @@ class LDrawLoaderService extends BaseLoaderService
     /**
      * @var Filesystem
      */
-    private $ldrawLibraryFilesystem;
+    private $ldrawLibrary;
 
     /**
-     * @var string download URL with current LDraw library
+     * @var Filesystem
      */
-    private $ldraw_url;
+    private $fileContext;
 
     /**
      * @var LDViewService
@@ -40,36 +38,62 @@ class LDrawLoaderService extends BaseLoaderService
     /** @var DatParser */
     private $datParser;
 
+    private $newModels;
+
+    /** @var RelationMapper */
+    protected $relationMapper;
+
     /**
      * LDrawLoaderService constructor.
      * @param Filesystem $ldrawLibraryFilesystem
      * @param LDViewService $LDViewService
-     * @param $ldraw_url
      * @param LDrawManager $ldrawService
-     * @param DatParser $datParser
+     * @param RelationMapper $relationMapper
      */
-    public function __construct($ldrawLibraryFilesystem, $LDViewService, $ldraw_url, $ldrawService, $datParser)
+    public function __construct($ldrawLibraryFilesystem, $LDViewService, $ldrawService, $relationMapper)
     {
         $this->LDViewService = $LDViewService;
-        $this->ldraw_url = $ldraw_url;
         $this->ldrawService = $ldrawService;
-        $this->datParser = $datParser;
-        $this->ldrawLibraryFilesystem = $ldrawLibraryFilesystem;
+        $this->datParser = new DatParser();
+        $this->ldrawLibrary = $ldrawLibraryFilesystem;
+        $this->relationMapper = $relationMapper;
+    }
+
+    private function dumpModel($model, $level = 1) {
+        $level = $level + 1;
+        dump(str_repeat("-", 2*$level).'> '.$model->getNumber());
+        foreach ($model->getSubparts() as $subpart) {
+            $this->dumpModel($subpart->getSubpart(), $level);
+        }
     }
 
     public function loadAllModels()
     {
-        $files = $this->ldrawLibraryFilesystem->get('parts')->getContents();
+        $files = $this->ldrawLibrary->get('parts')->getContents();
         $modelManager = $this->ldrawService->getModelManager();
 
         $this->initProgressBar(count($files));
 
         foreach ($files as $file) {
             if ($file['type'] == 'file' && $file['extension'] == 'dat') {
-                $model = $this->loadModel($this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix().$file['path']);
 
-                if($model)
-                    $modelManager->getRepository()->save($model);
+                $this->newModels = [];
+
+                $model = $this->loadModel($this->ldrawLibrary->getAdapter()->getPathPrefix().$file['path']);
+
+                if($model !== null) {
+//                    dump($model->getNumber());
+                    try {
+                        $modelManager->getRepository()->save($model);
+                    } catch (\Exception $exception) {
+                        dump($exception);
+//                        dump($model);
+
+                        $this->dumpModel($model);
+
+                        exit();
+                    }
+                }
             }
 
             $this->progressBar->advance();
@@ -86,26 +110,49 @@ class LDrawLoaderService extends BaseLoaderService
      */
     public function loadModel($file)
     {
+        $model = null;
+
         $modelManager = $this->ldrawService->getModelManager();
         $subpartManager = $this->ldrawService->getSubpartManager();
-        $aliasManager = $this->ldrawService->getAliasManager();
 
-        if($model = $modelManager->findByNumber(basename($file,'.dat'))) {
+        if(($model = $modelManager->findByNumber(basename($file,'.dat'))) || ($model = isset($this->newModels[basename($file,'.dat')]) ? $this->newModels[basename($file,'.dat')] : null)) {
+            $this->LDViewService->datToPng($file);
             return $model;
         }
 
-        $header = $this->datParser->parse($file);
+        try {
+            $header = $this->datParser->parse($file);
+        } catch (\Exception $e) {
+            dump($e);
+            return null;
+        }
 
         if ($this->isModelIncluded($header)) {
-            if (isset($header['parent']) && ($parent = $this->getModelParent($header['parent'])) && ($parentFile = $this->findModelFile($parent))) {
-                $parentModel = $this->loadModel($parentFile);
 
-                if ($parentModel) {
-                    $alias = $aliasManager->create($header['id'], $parentModel);
-                    $aliasManager->getRepository()->save($alias);
-                    $this->progressBar->advance();
+            if($this->relationMapper->find($header['id'], 'alias_model') !== $header['id']) {
+                $parentFile = $this->findModelFile($this->relationMapper->find($header['id'], 'alias_model'));
+            } else {
+                $parentFile = isset($header['parent']) && strpos($header['parent'], 's\\') === false ? $this->findModelFile($header['parent']) : null;
+            }
+
+            if ($parentFile) {
+                try {
+                    $parentModel = $this->loadModel($parentFile);
+
+                    if($parentModel) {
+                        $alias = new Alias();
+                        $alias->setNumber($header['id']);
+                        $alias->setModel($parentModel);
+                        $parentModel->addAlias($alias);
+
+                        $this->newModels[$parentModel->getNumber()] = $parentModel;
+                        return $parentModel;
+                    }
+                } catch (\Exception $e) {
+                    dump('b');
+                    dump($e->getMessage());
+                    return null;
                 }
-                return $parentModel;
             } else {
                 $model = $modelManager->create($header['id']);
                 $model->setName($header['name']);
@@ -119,19 +166,21 @@ class LDrawLoaderService extends BaseLoaderService
                 }
 
                 if (isset($header['subparts'])) {
-                    foreach ($header['subparts'] as $subpartId) {
-                        $subpartId = $this->getModelParent($subpartId);
+                    foreach ($header['subparts'] as $subpartId => $count) {
+                        if(strpos($subpartId, 's\\') === false) {
+                            if(($subpartFile = $this->findModelFile($subpartId)) != null) {
+                                try {
+                                    $subModel = $this->loadModel($subpartFile);
 
-                        $subModel = $modelManager->getRepository()->findOneBy(['number'=>$subpartId]);
-
-                        if(!$subModel && ($subpartFile = $this->findModelFile($subpartId)) != null) {
-                            $subModel = $this->loadModel($subpartFile);
-                        }
-
-                        if ($subModel) {
-                            $subpart = $subpartManager->create($model, $subModel);
-                            $subpartManager->getRepository()->save($subpart);
-                            $this->progressBar->advance();
+                                    if ($subModel) {
+                                        $subpart = $subpartManager->create($model,$subModel,$count);
+                                        $model->addSubpart($subpart);
+                                    }
+                                } catch (\Exception $e) {
+                                    dump('c');
+                                    dump($e->getMessage());
+                                }
+                            }
                         }
                     }
                 }
@@ -140,9 +189,11 @@ class LDrawLoaderService extends BaseLoaderService
                 $model->setModified($header['modified']);
                 $model->setPath($this->loadStlModel($file));
 
-                return $model;
+                $this->newModels[$model->getNumber()] = $model;
             }
         }
+
+        return $model;
     }
 
     /**
@@ -154,40 +205,20 @@ class LDrawLoaderService extends BaseLoaderService
      */
     private function findModelFile($id)
     {
-        $path = 'parts/'.strtolower($id).'.dat';
-        if ($this->ldrawLibraryFilesystem->has($path)) {
-            return $this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix().$path;
+        $filename = strtolower($id).'.dat';
+
+        if($this->fileContext && $this->fileContext->has($filename)) {
+            return $this->fileContext->getAdapter()->getPathPrefix().$filename;
+        } else if ($this->ldrawLibrary->has('parts/'.$filename)) {
+            return $this->ldrawLibrary->getAdapter()->getPathPrefix().'parts/'.$filename;
         }
 
         return null;
     }
 
-    /**
-     * Recursively load model parent id of model with $number id.
-     *
-     * @param string $number
-     *
-     * @return string
-     */
-    private function getModelParent($number)
-    {
-        if (($file = $this->findModelFile($number)) == null) {
-            return $number;
-        }
-
-        $header = $this->datParser->parse($file);
-
-        do {
-            $parent = isset($header['parent']) ? $header['parent'] : null;
-
-            if ($file = $this->findModelFile($parent)) {
-                $header = $this->datParser->parse($file);
-            } else {
-                break;
-            }
-        } while ($parent);
-
-        return $header['id'];
+    public function loadFileContext($file) {
+        $adapter = new Local(dirname(realpath($file)));
+        $this->fileContext = new Filesystem($adapter);
     }
 
     /**
@@ -201,10 +232,9 @@ class LDrawLoaderService extends BaseLoaderService
     {
         // Do not include sticker parts and incomplete parts
         if (
-            strpos($header['id'], 's') !== 0
-            && $header['type'] != 'Subpart'
+            $header['type'] != 'Subpart'
             && $header['type'] != 'Sticker'
-            && !(($header['type'] == 'Printed') && $this->findModelFile($header['parent']))
+//            && !(($header['type'] == 'Printed') && $this->findModelFile($header['parent']))
         ) {
             return true;
         }

src/AppBundle/Utils/DatParser.php

@@ -8,19 +8,6 @@ use Symfony\Component\Asset\Exception\LogicException;
 
 class DatParser
 {
-    /** @var RelationMapper */
-    protected $relationMapper;
-
-    /**
-     * DatParser constructor.
-     *
-     * @param RelationMapper $relationMapper
-     */
-    public function __construct($relationMapper)
-    {
-        $this->relationMapper = $relationMapper;
-    }
-
     /**
      * Parse LDraw .dat file header identifying model store data to array.
      *
@@ -93,18 +80,20 @@ class DatParser
                                 }
                             }
                         } elseif (strpos($line, '1 ') === 0) {
-                            $header['subparts'][] = $this->getAlias($line);
+                            $id = $this->getAlias($line);
+
+                            if(isset($header['subparts'][$id])) {
+                                $header['subparts'][$id] = $header['subparts'][$id] + 1;
+                            } else {
+                                $header['subparts'][$id] = 1;
+                            }
                         }
                     }
 
                     if ($this->isStickerShortcutPart($header['name'], $header['id'])) {
                         $header['type'] = 'Sticker';
-                    } elseif (($parent = $this->relationMapper->find($header['id'], 'alias_model')) != $header['id']) {
-                        $header['type'] = 'Alias';
-                        $header['subparts'] = null;
-                        $header['parent'] = $parent;
                     } elseif (isset($header['subparts']) && count($header['subparts']) == 1 && in_array($header['type'], ['Part Alias', 'Shortcut Physical_Colour', 'Shortcut Alias', 'Part Physical_Colour'])) {
-                        $header['parent'] = $header['subparts'][0];
+                        $header['parent'] = array_keys($header['subparts'])[0];
                         $header['subparts'] = null;
                     } elseif ($parent = $this->getPrintedParentId($header['id'])) {
                         $header['type'] = 'Printed';
@@ -118,19 +107,26 @@ class DatParser
                         $header['type'] = 'Obsolete/Subpart';
                     }
 
-                    $header['name'] = ltrim($header['name'], '~');
+                    if(!isset($header['type'])) {
+                        $header['type'] = 'Unknown';
+                    }
+
+                    if(!isset($header['modified'])) {
+                        $header['modified'] = null;
+                    }
+
+//                    $header['name'] = ltrim($header['name'], '~');
 
                     fclose($handle);
 
                     return $header;
                 }
             } catch (\Exception $exception) {
-                dump($exception->getMessage());
-
-                return null;
+                dump($exception);
+                throw new LogicException('Error parsing '.$file);
             }
         }
-        return null;
+        throw new LogicException('File not found '.$file);
     }
 
     /**
@@ -147,6 +143,8 @@ class DatParser
      */
     public function getAlias($line)
     {
+        if(preg_match('/^1 16 0 0 0 -1 0 0 0 1 0 0 0 1 (.*)\.(dat|DAT)$/', $line, $matches))
+            return null;
         if (preg_match('/^1(.*) (.*)\.(dat|DAT)$/', $line, $matches)) {
             return $matches[2];
         }