From d551040739ad331cc7eb394f05e4ac7bbc5f5ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20H=C3=BCbner?= Date: Fri, 10 Mar 2017 23:25:14 +0100 Subject: [PATCH] Refactor LDrawLoader --- app/config/services.yml | 13 +- .../Command/LoadLDRawLibraryCommand.php | 38 +++ src/AppBundle/Command/Loader/LDrawLoader.php | 187 ----------- .../Controller/DownloadController.php | 2 +- src/AppBundle/Loader/LDrawLoader.php | 303 ++++++++++++++++++ src/AppBundle/{Command => }/Loader/Loader.php | 16 +- 6 files changed, 365 insertions(+), 194 deletions(-) create mode 100644 src/AppBundle/Command/LoadLDRawLibraryCommand.php delete mode 100644 src/AppBundle/Command/Loader/LDrawLoader.php create mode 100644 src/AppBundle/Loader/LDrawLoader.php rename src/AppBundle/{Command => }/Loader/Loader.php (84%) diff --git a/app/config/services.yml b/app/config/services.yml index 244a52a..8c776ea 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -17,6 +17,11 @@ services: class: AppBundle\Service\CollectionService arguments: ['@doctrine.orm.entity_manager', '@manager.brickset','@manager.rebrickable'] + service.loader: + abstract: true + class: AppBundle\Loader\Loader + arguments: ['@doctrine.orm.entity_manager'] + service.ldview: class: AppBundle\Service\LDViewService arguments: ['%ldview_bin%', '@oneup_flysystem.ldraw_filesystem'] @@ -25,9 +30,11 @@ services: class: AppBundle\Command\Loader\RebrickableLoader arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable', '%rebrickable_url%' ] loader.ldraw: - class: AppBundle\Command\Loader\LDrawLoader - arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem', '%ldraw_url%'] - + class: AppBundle\Loader\LDrawLoader + calls: + - [setArguments, ['@service.ldview', '%ldraw_url%']] + parent: service.loader + app.form.filter_set: class: AppBundle\Form\FilterSetType arguments: ['@manager.brickset'] diff --git a/src/AppBundle/Command/LoadLDRawLibraryCommand.php b/src/AppBundle/Command/LoadLDRawLibraryCommand.php new file mode 100644 index 0000000..4a4fd93 --- /dev/null +++ b/src/AppBundle/Command/LoadLDRawLibraryCommand.php @@ -0,0 +1,38 @@ +setName('app:load:ldraw') + ->setDescription('Loads LDraw library parts') + ->setHelp('This command allows you to..') + ->addArgument('ldraw_path', InputArgument::OPTIONAL, 'Path to LDraw library folder'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $ldrawLoader = $this->getContainer()->get('loader.ldraw'); + $ldrawLoader->setOutput($output); + + //TODO log errors + + try { + if (($ldrawPath = $input->getArgument('ldraw_path')) == null) { + $ldrawPath = $ldrawLoader->downloadLibrary(); + } + + $ldrawLoader->loadModels($ldrawPath); + } catch (\Exception $e) { + printf($e->getMessage()); + } + } +} diff --git a/src/AppBundle/Command/Loader/LDrawLoader.php b/src/AppBundle/Command/Loader/LDrawLoader.php deleted file mode 100644 index 92dc37f..0000000 --- a/src/AppBundle/Command/Loader/LDrawLoader.php +++ /dev/null @@ -1,187 +0,0 @@ -em = $em; - $this->ldview = $ldview; - $this->dataPath = $dataPath; - $this->ldraw_url = $ldraw_url; - } - - public function downloadLibrary() - { - $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com'); - $temp = $this->downloadFile($this->ldraw_url); - $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.'); - if (file_exists($temp_dir)) { - unlink($temp_dir); - } - mkdir($temp_dir); - $zip = new \ZipArchive(); - if ($zip->open($temp) != 'true') { - echo 'Error :- Unable to open the Zip File'; - } - $zip->extractTo($temp_dir); - $zip->close(); - unlink($temp); - - return $temp_dir; - } - - public function loadModels($LDrawLibrary) - { - //TODO Refactor, use flysystem - $adapter = new Local(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary); - $this->ldraw = new Filesystem($adapter); -// $files = $this->ldraw->get('parts')->getContents(); - - $finder = new Finder(); - $files = $finder->files()->name('*.dat')->depth('== 0')->in(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary.DIRECTORY_SEPARATOR.'parts'); - - $progressBar = new ProgressBar($this->output, $files->count()); - $progressBar->setFormat('very_verbose'); - $progressBar->setMessage('Loading LDraw library models'); - $progressBar->setFormat('%message:6s% %current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)'); - $progressBar->start(); - foreach ($files as $file) { - $model = $this->loadPartHeader($file); - $model->setFile($this->createStlFile($file)->getPath()); - - $this->em->persist($model); - $this->em->flush(); - - $progressBar->advance(); - } - $progressBar->finish(); - } - - /** - * @param SplFileInfo $file - * - * @return Model - */ - private function loadPartHeader($file) - { - $handle = fopen($file->getRealPath(), 'r'); - if ($handle) { - $firstLine = false; - - $model = new Model(); - - // read lines while line starts with 0 or is empty - while (($line = trim(fgets($handle))) !== false && ($line ? $line[0] == '0' : true)) { - if ($line !== '') { - $line = preg_replace('/^0 /', '', $line); - - // 0 - if (!$firstLine) { - //TODO handle "~Moved to" - //TODO "=" - alias name for other part kept for referece - //TODO "_" shortcut - - $array = explode(' ', trim($line), 2); - $category = isset($array[0]) ? $array[0] : ''; - $model->setName($line); - - $firstLine = true; - } - // 0 !CATEGORY - elseif (strpos($line, '!CATEGORY ') === 0) { - $category = trim(preg_replace('/^!CATEGORY /', '', $line)); - } - // 0 !KEYWORDS , , ..., - elseif (strpos($line, '!KEYWORDS ') === 0) { - $keywords = explode(', ', preg_replace('/^!KEYWORDS /', '', $line)); - } - // 0 Name: .dat - elseif (strpos($line, 'Name: ') === 0) { - $model->setNumber(preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line)); - } - // 0 Author: [] - elseif (strpos($line, 'Author: ') === 0) { - $model->setAuthor(preg_replace('/^Author: /', '', $line)); - } - } - } - - $cat = $this->em->getRepository('AppBundle:Category')->findOneBy(['name' => $category]); - if ($cat == null) { - $cat = new Category(); - $cat->setName($category); - } - - $model->setCategory($cat); - $cat->addModel($model); - } else { - throw new LogicException('loadHeader error'); //TODO - } - fclose($handle); - - return $model; - } - - /** - * @param SplFileInfo $file - * - * @return \League\Flysystem\File - */ - private function createStlFile($file) - { - $stlFilename = str_replace('.dat', '.stl', $file->getFilename()); - - if (!$this->dataPath->has($stlFilename)) { - $builder = new ProcessBuilder(); - $process = $builder - ->setPrefix($this->ldview) - ->setArguments([ -// $this->ldraw->getAdapter()->getPathPrefix().$file['path'], - $file->getRealPath(), - '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(), - '-ExportFile='.$this->dataPath->getAdapter()->getPathPrefix().$stlFilename, - ]) - ->getProcess(); - - $process->run(); - - if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) { - throw new LogicException($file->getFilename().' : '.$process->getOutput()); //TODO - } - } - - return $this->dataPath->get($stlFilename); - } -} diff --git a/src/AppBundle/Controller/DownloadController.php b/src/AppBundle/Controller/DownloadController.php index 3447bcb..ce28760 100644 --- a/src/AppBundle/Controller/DownloadController.php +++ b/src/AppBundle/Controller/DownloadController.php @@ -2,7 +2,7 @@ namespace AppBundle\Controller; -use AppBundle\Entity\Model; +use AppBundle\Entity\LDraw\Model; use Symfony\Bundle\FrameworkBundle\Controller\Controller; use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException; diff --git a/src/AppBundle/Loader/LDrawLoader.php b/src/AppBundle/Loader/LDrawLoader.php new file mode 100644 index 0000000..7457699 --- /dev/null +++ b/src/AppBundle/Loader/LDrawLoader.php @@ -0,0 +1,303 @@ +LDViewService = $LDViewService; + $this->ldraw_url = $ldraw_url; + } + + public function downloadLibrary() + { + $this->output->writeln('Downloading LDraw library form ldraw.org'); + $temp = $this->downloadFile($this->ldraw_url); + $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.'); + if (file_exists($temp_dir)) { + unlink($temp_dir); + } + mkdir($temp_dir); + $zip = new \ZipArchive(); + if ($zip->open($temp) != 'true') { + echo 'Error :- Unable to open the Zip File'; + } + $zip->extractTo($temp_dir); + $zip->close(); + unlink($temp); + + return $temp_dir; + } + + public function loadModels($LDrawLibrary) + { + $adapter = new Local($LDrawLibrary); + $this->ldraw = new Filesystem($adapter); +// $files = $this->ldraw->get('parts')->getContents(); + + $files = $this->ldraw->get('parts')->getContents(); + + $this->em->getConnection()->getConfiguration()->setSQLLogger(null); + +// $finder = new Finder(); +// $files = $finder->files()->name('*.dat')->depth('== 0')->in(getcwd().DIRECTORY_SEPARATOR.$LDrawLibrary.DIRECTORY_SEPARATOR.'parts'); + + $progressBar = new ProgressBar($this->output, count($files)); + $progressBar->setFormat('very_verbose'); + $progressBar->setMessage('Loading LDraw library models'); + $progressBar->setFormat('%message:6s% %current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)'); + $progressBar->start(); + + foreach ($files as $file) { + if ($file['type'] == 'file' && $file['extension'] == 'dat') { + $header = $this->getPartHeader($file); + + if ($this->fileFilter($header)) { + if (null == ($part = $this->em->getRepository(Part::class)->find($header['id']))) { + $part = new Part(); + $part->setId($header['id']); + } + $part->setName($header['name']); + + if (($category = $this->em->getRepository(Category::class)->findOneBy(['name' => $header['category']])) == null) { + $category = new Category(); + $category->setName($header['category']); + } + $part->setCategory($category); + + if (($type = $this->em->getRepository(Type::class)->findOneBy(['name' => $header['type']])) == null) { + $type = new Type(); + $type->setName($header['type']); + } + $part->setType($type); + + if (isset($header['keywords'])) { + foreach ($header['keywords'] as $kword) { + $kword = trim($kword); + if (($keyword = $this->em->getRepository(Keyword::class)->findOneBy(['name' => $kword])) == null) { + $keyword = new Keyword(); + $keyword->setName($kword); + } + $part->addKeyword($keyword); + } + } + + if ($header['print_of_id']) { + if (($printParent = $this->em->getRepository(Part::class)->find($header['print_of_id'])) == null) { + $printParent = new Part(); + $printParent->setId($header['print_of_id']); + } + $part->setPrintOf($printParent); + } + + if ($header['alias_of_id']) { + if (($aliasParent = $this->em->getRepository(Part::class)->find($header['alias_of_id'])) == null) { + $aliasParent = new Part(); + $aliasParent->setId($header['alias_of_id']); + } + $part->setAliasOf($aliasParent); + } + + if (!$header['print_of_id'] && !$header['alias_of_id']) { + if (($model = $this->em->getRepository(Model::class)->find($header['id'])) == null) { + $model = new Model(); + $model->setId($header['id']); + } + + $model->setAuthor($header['author']); + $model->setModified($header['modified']); + + try { + $file = $this->LDViewService->datToStl($file, $this->ldraw)->getPath(); + } catch (\Exception $e) { + dump($e); + } + + $model->setFile($file); + + $part->setModel($model); + } + + $this->em->persist($part); + $this->em->flush(); + $this->em->clear(); + } + } + $progressBar->advance(); + } + $progressBar->finish(); + } + + private function fileFilter($header) + { + if (strpos($header['name'], 'Sticker') !== 0 && + (strpos($header['name'], '~') !== 0 || strpos($header['name'], '~Moved to ') === 0) && + $header['type'] !== 'Subpart') { + if (strpos($header['name'], '~Moved to ') === 0) { + $filepath = str_replace('\\', DIRECTORY_SEPARATOR, 'parts/'.$header['alias_of_id'].'.dat'); + + return $this->fileFilter($this->getPartHeader($this->ldraw->get($filepath)->getMetadata())); + } + + return true; + } + + return false; + } + + private function getPrinetedParentId($filename) + { + // nnnPxx, nnnnPxx, nnnnnPxx, nnnaPxx, nnnnaPxx + // where (a = alpha, n= numeric, x = alphanumeric) + + if (preg_match('/(^.*)(p[0-9a-z][0-9a-z][0-9a-z]{0,1})$/', $filename, $matches)) { + return $matches[1]; + } + + return null; + } + + private function isShortcutPart($filename) + { + // nnnCnn, nnnnCnn, nnnnnCnn Shortcut assembly of part nnn, nnnn or nnnnn with other parts or formed version of flexible part nnn, nnnn or nnnnn. + // nnnCnn-Fn, nnnnCnn-Fn, nnnnnCnn-Fn Positional variant of shortcut assembly of movable parts, comprising part nnn, nnnn or nnnnn with other parts. + // where (a = alpha, n= numeric, x = alphanumeric) + + return preg_match('/(^.*)(c[0-9][0-9])(.*)/', $filename); + } + + private function getAliasParentId($name) + { + if (preg_match('/^(~Moved to )(.*)$/', $name, $matches)) { + return $matches[2]; + } + + return null; + } + + private function getAlias($line) + { + // 1 x y z a b c d e f g h i + + if (preg_match('/^1(.*) (.*)\.dat$/', $line, $matches)) { + return $matches[2]; + } + + return null; + } + + /** + * @return array + */ + private function getPartHeader($file) + { + $header = []; + +// $handle = fopen($file->getRealPath(), 'r'); + + $handle = $this->ldraw->readStream($file['path']); + + if ($handle) { + $firstLine = false; + + while (($line = fgets($handle)) !== false) { + $line = trim($line); + + // Comments or META Commands + if (strpos($line, '0 ') === 0) { + $line = preg_replace('/^0 /', '', $line); + + // 0 + if (!$firstLine) { + $array = explode(' ', trim($line), 2); + $header['category'] = isset($array[0]) ? ltrim($array[0], '=_~') : ''; + $header['name'] = $line; + + $firstLine = true; + } + // 0 !CATEGORY + elseif (strpos($line, '!CATEGORY ') === 0) { + $header['category'] = trim(preg_replace('/^!CATEGORY /', '', $line)); + } + // 0 !KEYWORDS , , ..., + elseif (strpos($line, '!KEYWORDS ') === 0) { + $header['keywords'] = explode(', ', preg_replace('/^!KEYWORDS /', '', $line)); + } + // 0 Name: .dat + elseif (strpos($line, 'Name: ') === 0) { + $header['id'] = preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line); + } + // 0 Author: [] + elseif (strpos($line, 'Author: ') === 0) { + $header['author'] = preg_replace('/^Author: /', '', $line); + } + // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR + elseif (strpos($line, '!LDRAW_ORG ') === 0) { + $type = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE| ORIGINAL)(.*)/', '$2', $line); + $header['type'] = $type; + + // Last modification date in format YYYY-RR + $date = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE | ORIGINAL )(.*)/', '$4', $line); + if (preg_match('/^[1-2][0-9]{3}-[0-9]{2}$/', $date)) { + $header['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date.'-01 00:00:00'); + } else { + $header['modified'] = null; + } + } + } elseif (strpos($line, '1 ') === 0) { + if ($header['type'] == 'Part Alias' || $header['type'] == 'Shortcut Physical_Colour' || $header['type'] == 'Shortcut Alias') { + // "=" -> Alias name for other part kept for referece - do not include model -> LINK + // "_" -> Physical_color - do not include model -> LINK + $header['name'] = ltrim($header['name'], '=_'); + $header['alias_of_id'] = $this->getAlias($line); + } elseif ($header['type'] == 'Shortcut') { + $header['subparts'][] = $this->getAlias($line); + } + } elseif ($line != '') { + break; + } + } + + if (isset($header['id'])) { + $header['print_of_id'] = $this->getPrinetedParentId($header['id']); + } + + if (isset($header['name']) && !isset($header['alias_of_id'])) { + $header['alias_of_id'] = $this->getAliasParentId($header['name']); + } + + fclose($handle); + + return $header; + } + throw new LogicException('loadHeader error'); //TODO + } +} diff --git a/src/AppBundle/Command/Loader/Loader.php b/src/AppBundle/Loader/Loader.php similarity index 84% rename from src/AppBundle/Command/Loader/Loader.php rename to src/AppBundle/Loader/Loader.php index f0f2622..41966d2 100644 --- a/src/AppBundle/Command/Loader/Loader.php +++ b/src/AppBundle/Loader/Loader.php @@ -1,6 +1,6 @@ em = $em; + } + public function setOutput(OutputInterface $output) { $this->output = $output; $this->output->setDecorated(true); } - private function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max) + protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max) { switch ($notification_code) { case STREAM_NOTIFY_FILE_SIZE_IS: