Refactor LDrawLoader

app/config/services.yml

@@ -17,6 +17,11 @@ services:
         class: AppBundle\Service\CollectionService
         arguments: ['@doctrine.orm.entity_manager', '@manager.brickset','@manager.rebrickable']
 
+    service.loader:
+        abstract:  true
+        class: AppBundle\Loader\Loader
+        arguments: ['@doctrine.orm.entity_manager']
+
     service.ldview:
         class: AppBundle\Service\LDViewService
         arguments: ['%ldview_bin%', '@oneup_flysystem.ldraw_filesystem']
@@ -25,9 +30,11 @@ services:
         class: AppBundle\Command\Loader\RebrickableLoader
         arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable', '%rebrickable_url%' ]
     loader.ldraw:
-        class: AppBundle\Command\Loader\LDrawLoader
-        arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem', '%ldraw_url%']
-  
+        class: AppBundle\Loader\LDrawLoader
+        calls:
+          - [setArguments, ['@service.ldview', '%ldraw_url%']]
+        parent: service.loader
+
     app.form.filter_set:
         class: AppBundle\Form\FilterSetType
         arguments: ['@manager.brickset']

src/AppBundle/Command/LoadLDRawLibraryCommand.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace AppBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class LoadLDRawLibraryCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('app:load:ldraw')
+            ->setDescription('Loads LDraw library parts')
+            ->setHelp('This command allows you to..')
+            ->addArgument('ldraw_path', InputArgument::OPTIONAL, 'Path to LDraw library folder');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $ldrawLoader = $this->getContainer()->get('loader.ldraw');
+        $ldrawLoader->setOutput($output);
+
+        //TODO log errors
+
+        try {
+            if (($ldrawPath = $input->getArgument('ldraw_path')) == null) {
+                $ldrawPath = $ldrawLoader->downloadLibrary();
+            }
+
+            $ldrawLoader->loadModels($ldrawPath);
+        } catch (\Exception $e) {
+            printf($e->getMessage());
+        }
+    }
+}

src/AppBundle/Command/Loader/LDrawLoader.php

@@ -1,187 +0,0 @@
-<?php
-
-namespace AppBundle\Command\Loader;
-
-use AppBundle\Entity\Category;
-use AppBundle\Entity\Model;
-use League\Flysystem\Adapter\Local;
-use League\Flysystem\Filesystem;
-use Symfony\Component\Asset\Exception\LogicException;
-use Symfony\Component\Console\Helper\ProgressBar;
-use Symfony\Component\Finder\Finder;
-use Symfony\Component\Finder\SplFileInfo;
-use Symfony\Component\Process\ProcessBuilder;
-
-class LDrawLoader extends Loader
-{
-    /**
-     * @var string LDView binary file path
-     */
-    private $ldview;
-
-    /**
-     * @var Filesystem
-     */
-    private $ldraw;
-
-    /**
-     * @var \League\Flysystem\Filesystem
-     */
-    private $dataPath;
-
-    private $ldraw_url;
-
-    public function __construct($em, $ldview, $dataPath, $ldraw_url)
-    {
-        /*
-         * @var $em EntityManager
-         * */
-        $this->em = $em;
-        $this->ldview = $ldview;
-        $this->dataPath = $dataPath;
-        $this->ldraw_url = $ldraw_url;
-    }
-
-    public function downloadLibrary()
-    {
-        $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com');
-        $temp = $this->downloadFile($this->ldraw_url);
-        $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.');
-        if (file_exists($temp_dir)) {
-            unlink($temp_dir);
-        }
-        mkdir($temp_dir);
-        $zip = new \ZipArchive();
-        if ($zip->open($temp) != 'true') {
-            echo 'Error :- Unable to open the Zip File';
-        }
-        $zip->extractTo($temp_dir);
-        $zip->close();
-        unlink($temp);
-
-        return $temp_dir;
-    }
-
-    public function loadModels($LDrawLibrary)
-    {
-        //TODO Refactor, use flysystem
-        $adapter = new Local(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary);
-        $this->ldraw = new Filesystem($adapter);
-//        $files = $this->ldraw->get('parts')->getContents();
-
-        $finder = new Finder();
-        $files = $finder->files()->name('*.dat')->depth('== 0')->in(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary.DIRECTORY_SEPARATOR.'parts');
-
-        $progressBar = new ProgressBar($this->output, $files->count());
-        $progressBar->setFormat('very_verbose');
-        $progressBar->setMessage('Loading LDraw library models');
-        $progressBar->setFormat('%message:6s% %current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)');
-        $progressBar->start();
-        foreach ($files as $file) {
-            $model = $this->loadPartHeader($file);
-            $model->setFile($this->createStlFile($file)->getPath());
-
-            $this->em->persist($model);
-            $this->em->flush();
-
-            $progressBar->advance();
-        }
-        $progressBar->finish();
-    }
-
-    /**
-     * @param SplFileInfo $file
-     *
-     * @return Model
-     */
-    private function loadPartHeader($file)
-    {
-        $handle = fopen($file->getRealPath(), 'r');
-        if ($handle) {
-            $firstLine = false;
-
-            $model = new Model();
-
-            // read lines while line starts with 0 or is empty
-            while (($line = trim(fgets($handle))) !== false && ($line ? $line[0] == '0' : true)) {
-                if ($line !== '') {
-                    $line = preg_replace('/^0 /', '', $line);
-
-                    // 0 <CategoryName> <PartDescription>
-                    if (!$firstLine) {
-                        //TODO handle "~Moved to"
-                        //TODO        "=" - alias name for other part kept for referece
-                        //TODO        "_" shortcut
-
-                        $array = explode(' ', trim($line), 2);
-                        $category = isset($array[0]) ? $array[0] : '';
-                        $model->setName($line);
-
-                        $firstLine = true;
-                    }
-                    // 0 !CATEGORY <CategoryName>
-                    elseif (strpos($line, '!CATEGORY ') === 0) {
-                        $category = trim(preg_replace('/^!CATEGORY /', '', $line));
-                    }
-                    // 0 !KEYWORDS <first keyword>, <second keyword>, ..., <last keyword>
-                    elseif (strpos($line, '!KEYWORDS ') === 0) {
-                        $keywords = explode(', ', preg_replace('/^!KEYWORDS /', '', $line));
-                    }
-                    // 0 Name: <Filename>.dat
-                    elseif (strpos($line, 'Name: ') === 0) {
-                        $model->setNumber(preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line));
-                    }
-                    // 0 Author: <Realname> [<Username>]
-                    elseif (strpos($line, 'Author: ') === 0) {
-                        $model->setAuthor(preg_replace('/^Author: /', '', $line));
-                    }
-                }
-            }
-
-            $cat = $this->em->getRepository('AppBundle:Category')->findOneBy(['name' => $category]);
-            if ($cat == null) {
-                $cat = new Category();
-                $cat->setName($category);
-            }
-
-            $model->setCategory($cat);
-            $cat->addModel($model);
-        } else {
-            throw new LogicException('loadHeader error'); //TODO
-        }
-        fclose($handle);
-
-        return $model;
-    }
-
-    /**
-     * @param SplFileInfo $file
-     *
-     * @return \League\Flysystem\File
-     */
-    private function createStlFile($file)
-    {
-        $stlFilename = str_replace('.dat', '.stl', $file->getFilename());
-
-        if (!$this->dataPath->has($stlFilename)) {
-            $builder = new ProcessBuilder();
-            $process = $builder
-                ->setPrefix($this->ldview)
-                ->setArguments([
-//                $this->ldraw->getAdapter()->getPathPrefix().$file['path'],
-                    $file->getRealPath(),
-                    '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(),
-                    '-ExportFile='.$this->dataPath->getAdapter()->getPathPrefix().$stlFilename,
-                ])
-                ->getProcess();
-
-            $process->run();
-
-            if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) {
-                throw new LogicException($file->getFilename().' : '.$process->getOutput()); //TODO
-            }
-        }
-
-        return $this->dataPath->get($stlFilename);
-    }
-}

src/AppBundle/Controller/DownloadController.php

@@ -2,7 +2,7 @@
 
 namespace AppBundle\Controller;
 
-use AppBundle\Entity\Model;
+use AppBundle\Entity\LDraw\Model;
 use Symfony\Bundle\FrameworkBundle\Controller\Controller;
 use Symfony\Component\HttpFoundation\BinaryFileResponse;
 use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;

src/AppBundle/Loader/LDrawLoader.php

@@ -0,0 +1,303 @@
+<?php
+
+namespace AppBundle\Loader;
+
+use AppBundle\Entity\LDraw\Category;
+use AppBundle\Entity\LDraw\Keyword;
+use AppBundle\Entity\LDraw\Model;
+use AppBundle\Entity\LDraw\Part;
+use AppBundle\Entity\LDraw\Type;
+use AppBundle\Service\LDViewService;
+use League\Flysystem\Adapter\Local;
+use League\Flysystem\Filesystem;
+use Symfony\Component\Asset\Exception\LogicException;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Finder\Finder;
+
+class LDrawLoader extends Loader
+{
+    /**
+     * @var Filesystem
+     */
+    private $ldraw;
+
+    private $ldraw_url;
+
+    /**
+     * @var LDViewService
+     */
+    private $LDViewService;
+
+    /**
+     * @param array $ldraw_url
+     */
+    public function setArguments(LDViewService $LDViewService, $ldraw_url)
+    {
+        $this->LDViewService = $LDViewService;
+        $this->ldraw_url = $ldraw_url;
+    }
+
+    public function downloadLibrary()
+    {
+        $this->output->writeln('Downloading LDraw library form ldraw.org');
+        $temp = $this->downloadFile($this->ldraw_url);
+        $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.');
+        if (file_exists($temp_dir)) {
+            unlink($temp_dir);
+        }
+        mkdir($temp_dir);
+        $zip = new \ZipArchive();
+        if ($zip->open($temp) != 'true') {
+            echo 'Error :- Unable to open the Zip File';
+        }
+        $zip->extractTo($temp_dir);
+        $zip->close();
+        unlink($temp);
+
+        return $temp_dir;
+    }
+
+    public function loadModels($LDrawLibrary)
+    {
+        $adapter = new Local($LDrawLibrary);
+        $this->ldraw = new Filesystem($adapter);
+//        $files = $this->ldraw->get('parts')->getContents();
+
+        $files = $this->ldraw->get('parts')->getContents();
+
+        $this->em->getConnection()->getConfiguration()->setSQLLogger(null);
+
+//        $finder = new Finder();
+//        $files = $finder->files()->name('*.dat')->depth('== 0')->in(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary.DIRECTORY_SEPARATOR.'parts');
+
+        $progressBar = new ProgressBar($this->output, count($files));
+        $progressBar->setFormat('very_verbose');
+        $progressBar->setMessage('Loading LDraw library models');
+        $progressBar->setFormat('%message:6s% %current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)');
+        $progressBar->start();
+
+        foreach ($files as $file) {
+            if ($file['type'] == 'file' && $file['extension'] == 'dat') {
+                $header = $this->getPartHeader($file);
+
+                if ($this->fileFilter($header)) {
+                    if (null == ($part = $this->em->getRepository(Part::class)->find($header['id']))) {
+                        $part = new Part();
+                        $part->setId($header['id']);
+                    }
+                    $part->setName($header['name']);
+
+                    if (($category = $this->em->getRepository(Category::class)->findOneBy(['name' => $header['category']])) == null) {
+                        $category = new Category();
+                        $category->setName($header['category']);
+                    }
+                    $part->setCategory($category);
+
+                    if (($type = $this->em->getRepository(Type::class)->findOneBy(['name' => $header['type']])) == null) {
+                        $type = new Type();
+                        $type->setName($header['type']);
+                    }
+                    $part->setType($type);
+
+                    if (isset($header['keywords'])) {
+                        foreach ($header['keywords'] as $kword) {
+                            $kword = trim($kword);
+                            if (($keyword = $this->em->getRepository(Keyword::class)->findOneBy(['name' => $kword])) == null) {
+                                $keyword = new Keyword();
+                                $keyword->setName($kword);
+                            }
+                            $part->addKeyword($keyword);
+                        }
+                    }
+
+                    if ($header['print_of_id']) {
+                        if (($printParent = $this->em->getRepository(Part::class)->find($header['print_of_id'])) == null) {
+                            $printParent = new Part();
+                            $printParent->setId($header['print_of_id']);
+                        }
+                        $part->setPrintOf($printParent);
+                    }
+
+                    if ($header['alias_of_id']) {
+                        if (($aliasParent = $this->em->getRepository(Part::class)->find($header['alias_of_id'])) == null) {
+                            $aliasParent = new Part();
+                            $aliasParent->setId($header['alias_of_id']);
+                        }
+                        $part->setAliasOf($aliasParent);
+                    }
+
+                    if (!$header['print_of_id'] && !$header['alias_of_id']) {
+                        if (($model = $this->em->getRepository(Model::class)->find($header['id'])) == null) {
+                            $model = new Model();
+                            $model->setId($header['id']);
+                        }
+
+                        $model->setAuthor($header['author']);
+                        $model->setModified($header['modified']);
+
+                        try {
+                            $file = $this->LDViewService->datToStl($file, $this->ldraw)->getPath();
+                        } catch (\Exception $e) {
+                            dump($e);
+                        }
+
+                        $model->setFile($file);
+
+                        $part->setModel($model);
+                    }
+
+                    $this->em->persist($part);
+                    $this->em->flush();
+                    $this->em->clear();
+                }
+            }
+            $progressBar->advance();
+        }
+        $progressBar->finish();
+    }
+
+    private function fileFilter($header)
+    {
+        if (strpos($header['name'], 'Sticker') !== 0 &&
+            (strpos($header['name'], '~') !== 0 || strpos($header['name'], '~Moved to ') === 0) &&
+            $header['type'] !== 'Subpart') {
+            if (strpos($header['name'], '~Moved to ') === 0) {
+                $filepath = str_replace('\\', DIRECTORY_SEPARATOR, 'parts/'.$header['alias_of_id'].'.dat');
+
+                return $this->fileFilter($this->getPartHeader($this->ldraw->get($filepath)->getMetadata()));
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private function getPrinetedParentId($filename)
+    {
+        // nnnPxx, nnnnPxx, nnnnnPxx, nnnaPxx, nnnnaPxx
+        //  where (a = alpha, n= numeric, x = alphanumeric)
+
+        if (preg_match('/(^.*)(p[0-9a-z][0-9a-z][0-9a-z]{0,1})$/', $filename, $matches)) {
+            return $matches[1];
+        }
+
+        return null;
+    }
+
+    private function isShortcutPart($filename)
+    {
+        // nnnCnn, nnnnCnn, nnnnnCnn	        Shortcut assembly of part nnn, nnnn or nnnnn with other parts or formed version of flexible part nnn, nnnn or nnnnn.
+        // nnnCnn-Fn, nnnnCnn-Fn, nnnnnCnn-Fn   Positional variant of shortcut assembly of movable parts, comprising part nnn, nnnn or nnnnn with other parts.
+        //  where (a = alpha, n= numeric, x = alphanumeric)
+
+        return preg_match('/(^.*)(c[0-9][0-9])(.*)/', $filename);
+    }
+
+    private function getAliasParentId($name)
+    {
+        if (preg_match('/^(~Moved to )(.*)$/', $name, $matches)) {
+            return $matches[2];
+        }
+
+        return null;
+    }
+
+    private function getAlias($line)
+    {
+        //        1 <colour> x y z a b c d e f g h i <file>
+
+        if (preg_match('/^1(.*) (.*)\.dat$/', $line, $matches)) {
+            return $matches[2];
+        }
+
+        return null;
+    }
+
+    /**
+     * @return array
+     */
+    private function getPartHeader($file)
+    {
+        $header = [];
+
+//        $handle = fopen($file->getRealPath(), 'r');
+
+        $handle = $this->ldraw->readStream($file['path']);
+
+        if ($handle) {
+            $firstLine = false;
+
+            while (($line = fgets($handle)) !== false) {
+                $line = trim($line);
+
+                // Comments or META Commands
+                if (strpos($line, '0 ') === 0) {
+                    $line = preg_replace('/^0 /', '', $line);
+
+                    // 0 <CategoryName> <PartDescription>
+                    if (!$firstLine) {
+                        $array = explode(' ', trim($line), 2);
+                        $header['category'] = isset($array[0]) ? ltrim($array[0], '=_~') : '';
+                        $header['name'] = $line;
+
+                        $firstLine = true;
+                    }
+                    // 0 !CATEGORY <CategoryName>
+                    elseif (strpos($line, '!CATEGORY ') === 0) {
+                        $header['category'] = trim(preg_replace('/^!CATEGORY /', '', $line));
+                    }
+                    // 0 !KEYWORDS <first keyword>, <second keyword>, ..., <last keyword>
+                    elseif (strpos($line, '!KEYWORDS ') === 0) {
+                        $header['keywords'] = explode(', ', preg_replace('/^!KEYWORDS /', '', $line));
+                    }
+                    // 0 Name: <Filename>.dat
+                    elseif (strpos($line, 'Name: ') === 0) {
+                        $header['id'] = preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line);
+                    }
+                    // 0 Author: <Realname> [<Username>]
+                    elseif (strpos($line, 'Author: ') === 0) {
+                        $header['author'] = preg_replace('/^Author: /', '', $line);
+                    }
+                    // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR
+                    elseif (strpos($line, '!LDRAW_ORG ') === 0) {
+                        $type = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE| ORIGINAL)(.*)/', '$2', $line);
+                        $header['type'] = $type;
+
+                        // Last modification date in format YYYY-RR
+                        $date = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE | ORIGINAL )(.*)/', '$4', $line);
+                        if (preg_match('/^[1-2][0-9]{3}-[0-9]{2}$/', $date)) {
+                            $header['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date.'-01 00:00:00');
+                        } else {
+                            $header['modified'] = null;
+                        }
+                    }
+                } elseif (strpos($line, '1 ') === 0) {
+                    if ($header['type'] == 'Part Alias' || $header['type'] == 'Shortcut Physical_Colour' || $header['type'] == 'Shortcut Alias') {
+                        // "=" -> Alias name for other part kept for referece - do not include model -> LINK
+                        // "_" -> Physical_color  - do not include model -> LINK
+                        $header['name'] = ltrim($header['name'], '=_');
+                        $header['alias_of_id'] = $this->getAlias($line);
+                    } elseif ($header['type'] == 'Shortcut') {
+                        $header['subparts'][] = $this->getAlias($line);
+                    }
+                } elseif ($line != '') {
+                    break;
+                }
+            }
+
+            if (isset($header['id'])) {
+                $header['print_of_id'] = $this->getPrinetedParentId($header['id']);
+            }
+
+            if (isset($header['name']) && !isset($header['alias_of_id'])) {
+                $header['alias_of_id'] = $this->getAliasParentId($header['name']);
+            }
+
+            fclose($handle);
+
+            return $header;
+        }
+        throw new LogicException('loadHeader error'); //TODO
+    }
+}

src/AppBundle/Loader/Loader.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace AppBundle\Command\Loader;
+namespace AppBundle\Loader;
 
 use Doctrine\ORM\EntityManager;
 use Symfony\Component\Asset\Exception\LogicException;
@@ -8,7 +8,7 @@ use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Debug\Exception\ContextErrorException;
 
-class Loader
+abstract class Loader
 {
     /**
      * @var EntityManager
@@ -25,13 +25,23 @@ class Loader
      */
     protected $progressBar;
 
+    /**
+     * Loader constructor.
+     *
+     * @param EntityManager $em
+     */
+    public function __construct(EntityManager $em)
+    {
+        $this->em = $em;
+    }
+
     public function setOutput(OutputInterface $output)
     {
         $this->output = $output;
         $this->output->setDecorated(true);
     }
 
-    private function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
+    protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
     {
         switch ($notification_code) {
             case STREAM_NOTIFY_FILE_SIZE_IS: