From 8a1a5e2cff4a0eb3966d7687545a428960a7f20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20H=C3=BCbner?= Date: Sun, 16 Apr 2017 22:15:54 +0200 Subject: [PATCH] Improve load models command, log loading process --- README.md | 5 +- app/config/config_dev.yml | 9 +- app/config/service/loader.yml | 22 +- src/AppBundle/Command/LoadModelsCommand.php | 58 ++- .../Exception/ErrorParsingLineException.php | 15 + src/AppBundle/Exception/FileException.php | 17 + .../Exception/FileNotFoundException.php | 10 +- .../Exception/ParseErrorException.php | 29 +- .../Exception/WriteErrorException.php | 15 + src/AppBundle/Service/LDViewService.php | 55 ++- .../{BaseLoaderService.php => BaseLoader.php} | 46 ++- src/AppBundle/Service/Loader/ModelLoader.php | 331 ++++++++++++++++++ .../Service/Loader/ModelLoaderService.php | 250 ------------- .../{DatParser.php => LDModelParser.php} | 68 ++-- 14 files changed, 554 insertions(+), 376 deletions(-) create mode 100644 src/AppBundle/Exception/ErrorParsingLineException.php create mode 100644 src/AppBundle/Exception/FileException.php create mode 100644 src/AppBundle/Exception/WriteErrorException.php rename src/AppBundle/Service/Loader/{BaseLoaderService.php => BaseLoader.php} (62%) create mode 100644 src/AppBundle/Service/Loader/ModelLoader.php delete mode 100644 src/AppBundle/Service/Loader/ModelLoaderService.php rename src/AppBundle/Utils/{DatParser.php => LDModelParser.php} (74%) diff --git a/README.md b/README.md index fb24ea2..67d510c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ For full requirements see Symfony 3.2 [docs](http://symfony.com/doc/3.2/referenc #### Database 1. Set application parameters in *app/config/parameters.yml* -2. Generate empty database by running command `$ php bin/console doctrine:database:create` -3. Load LDraw models into database by running commad `$ php bin/console app:load:ldraw [ldraw_dir_path]` +2. Generate empty database by running command `$ php bin/console doctrine:database:create` +3. Create database tables by running command `$ bin/console doctrine:schema:create` +3. Load LDraw models into database by running commad `$ php bin/console app:load:models [--all] [--file=FILE] [--update]` 4. Load Rebrickable data into database by running command `$ php bin/console app:load:rebrickable` 5. Load relations between LDraw models and Rebrickable parts by running command `$ php bin/console app:load:relation` \ No newline at end of file diff --git a/app/config/config_dev.yml b/app/config/config_dev.yml index 7eb4739..f4d3c25 100644 --- a/app/config/config_dev.yml +++ b/app/config/config_dev.yml @@ -12,15 +12,22 @@ web_profiler: intercept_redirects: false monolog: + channels: ['loader'] handlers: main: type: stream path: "%kernel.logs_dir%/%kernel.environment%.log" level: debug - channels: [!event] + channels: [!event, !loader] console: type: console channels: [!event, !doctrine] + loader: + type: rotating_file + path: "%kernel.logs_dir%/loader.log" + level: debug + channels: 'loader' + max_files: 10 # uncomment to get logging in your browser # you may have to allow bigger header sizes in your Web server configuration #firephp: diff --git a/app/config/service/loader.yml b/app/config/service/loader.yml index 30b4804..0dbf6f6 100644 --- a/app/config/service/loader.yml +++ b/app/config/service/loader.yml @@ -1,25 +1,25 @@ services: - service.loader: + service.loader.base: abstract: true - class: AppBundle\Service\Loader\BaseLoaderService + class: AppBundle\Service\Loader\BaseLoader calls: - - [setArguments, ['@doctrine.orm.entity_manager', '@app.relation.mapper']] + - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']] service.ldview: class: AppBundle\Service\LDViewService arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem'] service.loader.rebrickable: - class: AppBundle\Service\Loader\RebrickableLoaderService - arguments: ['%rebrickable_url%'] - parent: service.loader + class: AppBundle\Service\Loader\RebrickableLoader + arguments: ['%rebrickable_csv_url%'] + parent: service.loader.base service.loader.model: - class: AppBundle\Service\Loader\ModelLoaderService - arguments: ['@service.ldview', '@manager.ldraw', '@app.relation.mapper'] - parent: service.loader + class: AppBundle\Service\Loader\ModelLoader + arguments: ['@service.ldview', '@app.relation.mapper'] + parent: service.loader.base service.loader.relation: class: AppBundle\Service\Loader\RelationLoader - arguments: ['@manager.ldraw.model', '@repository.rebrickable.part', '@api.manager.rebrickable'] - parent: service.loader \ No newline at end of file + arguments: ['@api.manager.rebrickable', '@app.relation.mapper'] + parent: service.loader.base \ No newline at end of file diff --git a/src/AppBundle/Command/LoadModelsCommand.php b/src/AppBundle/Command/LoadModelsCommand.php index c8ac9f1..c115440 100644 --- a/src/AppBundle/Command/LoadModelsCommand.php +++ b/src/AppBundle/Command/LoadModelsCommand.php @@ -19,18 +19,18 @@ class LoadModelsCommand extends ContainerAwareCommand $this ->setName('app:load:models') ->setDescription('Loads LDraw library models into database') - ->setHelp('This command allows you to load LDraw library models into while converting .dat files to .stl') + ->setHelp('This command allows you to load LDraw library models into database while converting .dat files to .stl format.') ->setDefinition( new InputDefinition([ new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'), - new InputOption('images', 'i',InputOption::VALUE_NONE, 'Do you want to generate images of models?'), - new InputOption('all','a',InputOption::VALUE_NONE, 'Do you want to load whole LDraw libary?'), - new InputOption('file','f',InputOption::VALUE_REQUIRED, 'Path to DAT file that should be loaded into database') +// new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'), + new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'), + new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'), + new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'), ]) ); } - //TODO log errors protected function execute(InputInterface $input, OutputInterface $output) { if (!$this->lock()) { @@ -39,25 +39,49 @@ class LoadModelsCommand extends ContainerAwareCommand return 0; } - $ldrawLoader = $this->getContainer()->get('service.loader.model'); - $ldrawLoader->setOutput($output); - $ldrawLoader->setLDrawLibraryContext(realpath($input->getArgument('ldraw'))); + $modelLoader = $this->getContainer()->get('service.loader.model'); + $modelLoader->setOutput($output); + $modelLoader->setRewite($input->getOption('update')); - try { - if (($ldrawPath = $input->getOption('file')) != null) { - $ldrawLoader->loadFileContext(dirname(realpath($ldrawPath))); + $ldraw = $input->getArgument('ldraw'); - $model = $ldrawLoader->loadModel($ldrawPath); + if ($ldrawPath = realpath($ldraw)) { + $modelLoader->setLDrawLibraryContext($ldrawPath); - if($model !== null) { - $this->getContainer()->get('manager.ldraw.model')->getRepository()->save($model); + if (($path = $input->getOption('file')) != null) { + if ($file = realpath($path)) { + $output->writeln([ + 'Loading model', + 'path: '.$file, + '------------------------------------------------------------------------------', + ]); + + $modelLoader->loadOneModel($file); + + $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors(); + $errors = $errorCount ? ''.$errorCount.'' : '0'; + + $output->writeln(['Done with "'.$errors.'" errors.']); + } else { + $output->writeln("File $path not found"); } } + + // Load all models inside ldraw/parts directory if ($input->getOption('all')) { - $ldrawLoader->loadAllModels(); + $output->writeln([ + 'Loading models from LDraw library: '.$ldrawPath.'', + ]); + + $modelLoader->loadAllModels(); + + $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors(); + $errors = $errorCount ? ''.$errorCount.'' : '0'; + + $output->writeln(['Done with "'.$errors.'" errors.']); } - } catch (\Exception $e) { - printf($e->getMessage()); + } else { + $output->writeln($ldraw.' is not valid path'); } $this->release(); diff --git a/src/AppBundle/Exception/ErrorParsingLineException.php b/src/AppBundle/Exception/ErrorParsingLineException.php new file mode 100644 index 0000000..ac9005b --- /dev/null +++ b/src/AppBundle/Exception/ErrorParsingLineException.php @@ -0,0 +1,15 @@ +path = $path; + + parent::__construct($message, $code, $previous); + } +} diff --git a/src/AppBundle/Exception/FileNotFoundException.php b/src/AppBundle/Exception/FileNotFoundException.php index 3e12a65..6a278f4 100644 --- a/src/AppBundle/Exception/FileNotFoundException.php +++ b/src/AppBundle/Exception/FileNotFoundException.php @@ -2,8 +2,14 @@ namespace AppBundle\Exception; +use Throwable; -class FileNotFoundException extends \Symfony\Component\Filesystem\Exception\FileNotFoundException +class FileNotFoundException extends FileException { + public function __construct($path, $message = '', $code = 0, Throwable $previous = null) + { + $message = sprintf('File "%s" not found.', $path); -} \ No newline at end of file + parent::__construct($path, $message, $code, $previous); + } +} diff --git a/src/AppBundle/Exception/ParseErrorException.php b/src/AppBundle/Exception/ParseErrorException.php index cdfa3fb..9552965 100644 --- a/src/AppBundle/Exception/ParseErrorException.php +++ b/src/AppBundle/Exception/ParseErrorException.php @@ -2,31 +2,14 @@ namespace AppBundle\Exception; +use Throwable; -class ParseErrorException extends \Exception +class ParseErrorException extends FileException { - private $filepath; - - public function __construct($filepath = "", $message = "", $code = 0, Exception $previous = null) + public function __construct($path, $message = '', $code = 0, Throwable $previous = null) { - parent::__construct($message, $code, $previous); + $message = sprintf('Error parsing "%s" file.', $path); - $this->filepath = $filepath; + parent::__construct($path, $message, $code, $previous); } - - /** - * @return mixed - */ - public function getFilepath() - { - return $this->filepath; - } - - /** - * @param mixed $filepath - */ - public function setFilepath($filepath) - { - $this->filepath = $filepath; - } -} \ No newline at end of file +} diff --git a/src/AppBundle/Exception/WriteErrorException.php b/src/AppBundle/Exception/WriteErrorException.php new file mode 100644 index 0000000..9346cd5 --- /dev/null +++ b/src/AppBundle/Exception/WriteErrorException.php @@ -0,0 +1,15 @@ +ldrawLibraryFilesystem = $ldrawLibraryFilesystem; + $this->ldrawLibraryContext = $ldrawLibraryContext; } /** @@ -57,35 +52,36 @@ class LDViewService * * @param $file * - * @return File * @throws ConvertingFailedException + * + * @return File */ - public function datToStl($file) + public function datToStl($file, $rewrite = false) { if (!$this->mediaFilesystem->has('ldraw'.DIRECTORY_SEPARATOR.'models')) { $this->mediaFilesystem->createDir('ldraw'.DIRECTORY_SEPARATOR.'models'); } - $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'models'.DIRECTORY_SEPARATOR.basename($file,'.dat').'.stl'; + $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'models'.DIRECTORY_SEPARATOR.basename($file, '.dat').'.stl'; - if (!file_exists($newFile) || $this->rewrite) { + if (!$this->mediaFilesystem->has($newFile) || $rewrite) { $this->runLDView([ $file, - '-LDrawDir='.$this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix(), + '-LDrawDir='.$this->ldrawLibraryContext->getAdapter()->getPathPrefix(), '-ExportFiles=1', '-ExportSuffix=.stl', '-ExportsDir='.$this->mediaFilesystem->getAdapter()->getPathPrefix().'ldraw'.DIRECTORY_SEPARATOR.'models', ]); - - // Check if file created successfully - if (!$this->mediaFilesystem->has($newFile)) { - throw new ConvertingFailedException($newFile); + if ($this->mediaFilesystem->has($newFile)) { + return $this->mediaFilesystem->get($newFile); } + } else { + return $this->mediaFilesystem->get($newFile); } - return $this->mediaFilesystem->get($newFile); + throw new ConvertingFailedException($file, 'STL'); } /** @@ -94,21 +90,22 @@ class LDViewService * * @param $file * - * @return File * @throws ConvertingFailedException + * + * @return File */ - public function datToPng($file) + public function datToPng($file, $rewrite = false) { if (!$this->mediaFilesystem->has('ldraw'.DIRECTORY_SEPARATOR.'images')) { $this->mediaFilesystem->createDir('ldraw'.DIRECTORY_SEPARATOR.'images'); } - $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'images'.DIRECTORY_SEPARATOR.basename($file,'.dat').'.png'; + $newFile = 'ldraw'.DIRECTORY_SEPARATOR.'images'.DIRECTORY_SEPARATOR.basename($file, '.dat').'.png'; if (!$this->mediaFilesystem->has($newFile) || $this->rewrite) { $this->runLDView([ $file, - '-LDrawDir='.$this->ldrawLibraryFilesystem->getAdapter()->getPathPrefix(), + '-LDrawDir='.$this->ldrawLibraryContext->getAdapter()->getPathPrefix(), '-AutoCrop=0', '-SaveAlpha=0', '-BackgroundColor3=0xFFFFFF', @@ -126,12 +123,14 @@ class LDViewService ]); // Check if file created successfully - if (!$this->mediaFilesystem->has($newFile)) { - throw new ConvertingFailedException($newFile); + if ($this->mediaFilesystem->has($newFile)) { + return $this->mediaFilesystem->get($newFile); } + } else { + return $this->mediaFilesystem->get($newFile); } - return $this->mediaFilesystem->get($newFile); + throw new ConvertingFailedException($file, 'PNG'); } /** diff --git a/src/AppBundle/Service/Loader/BaseLoaderService.php b/src/AppBundle/Service/Loader/BaseLoader.php similarity index 62% rename from src/AppBundle/Service/Loader/BaseLoaderService.php rename to src/AppBundle/Service/Loader/BaseLoader.php index 0143da9..07343ee 100644 --- a/src/AppBundle/Service/Loader/BaseLoaderService.php +++ b/src/AppBundle/Service/Loader/BaseLoader.php @@ -2,15 +2,17 @@ namespace AppBundle\Service\Loader; -use AppBundle\Utils\RelationMapper; +use AppBundle\Exception\FileNotFoundException; +use AppBundle\Exception\WriteErrorException; use Doctrine\ORM\EntityManager; +use Monolog\Logger; use Symfony\Bundle\FrameworkBundle\Translation\Translator; use Symfony\Component\Asset\Exception\LogicException; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Debug\Exception\ContextErrorException; -abstract class BaseLoaderService +abstract class BaseLoader { /** * @var EntityManager @@ -27,8 +29,8 @@ abstract class BaseLoaderService */ protected $progressBar; - /** @var RelationMapper */ - protected $relationMapper; + /** @var Logger */ + protected $logger; /** * Loader constructor. @@ -36,11 +38,12 @@ abstract class BaseLoaderService * @param EntityManager $em * @param Translator $translator */ - public function setArguments(EntityManager $em, $relationMapper) + public function setArguments(EntityManager $em, $logger) { $this->em = $em; - $this->relationMapper = $relationMapper; $this->em->getConnection()->getConfiguration()->setSQLLogger(null); + + $this->logger = $logger; } public function setOutput(OutputInterface $output) @@ -49,11 +52,17 @@ abstract class BaseLoaderService $this->output->setDecorated(true); } + /** + * Initialize new progress bar. + * + * @param $total + */ protected function initProgressBar($total) { $this->progressBar = new ProgressBar($this->output, $total); - $this->progressBar->setFormat('very_verbose'); - $this->progressBar->setFormat('%current%/%max% [%bar%]%percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL); +// $this->progressBar->setFormat('very_verbose'); + $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL); + $this->progressBar->setBarWidth(70); $this->progressBar->start(); } @@ -62,20 +71,31 @@ abstract class BaseLoaderService switch ($notification_code) { case STREAM_NOTIFY_FILE_SIZE_IS: $this->initProgressBar($bytes_max); + $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL); break; case STREAM_NOTIFY_PROGRESS: $this->progressBar->setProgress($bytes_transferred); break; case STREAM_NOTIFY_COMPLETED: - $this->progressBar->setProgress($bytes_transferred); + $this->progressBar->setMessage('Done'); + $this->progressBar->setProgress($bytes_max); $this->progressBar->finish(); break; } } + /** + * Download file from $url, save it to system temp directory and return filepath. + * + * @param $url + * + * @throws FileNotFoundException + * + * @return bool|string + */ protected function downloadFile($url) { - $this->output->writeln('Downloading file from: '.$url.''); + $this->output->writeln('Loading file from: '.$url.''); $temp = tempnam(sys_get_temp_dir(), 'printabrick.'); $ctx = stream_context_create([], [ @@ -84,12 +104,12 @@ abstract class BaseLoaderService try { if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) { - throw new LogicException('error writing file'); //TODO + throw new WriteErrorException($temp); } } catch (ContextErrorException $e) { - throw new LogicException('wrong url'); //TODO + throw new FileNotFoundException($url); } catch (\Exception $e) { - throw new LogicException('exception: '.$e->getMessage()); //TODO + throw new LogicException($e); } return $temp; diff --git a/src/AppBundle/Service/Loader/ModelLoader.php b/src/AppBundle/Service/Loader/ModelLoader.php new file mode 100644 index 0000000..6e47507 --- /dev/null +++ b/src/AppBundle/Service/Loader/ModelLoader.php @@ -0,0 +1,331 @@ +LDViewService = $LDViewService; + $this->relationMapper = $relationMapper; + $this->ldModelParser = new LDModelParser(); + $this->finder = new Finder(); + } + + /** + * @param bool $rewite + */ + public function setRewite($rewite) + { + $this->rewite = $rewite; + } + + /** + * @param $ldrawLibrary + */ + public function setLDrawLibraryContext($ldrawLibrary) + { + try { + $adapter = new Local($ldrawLibrary); + $this->ldrawLibraryContext = new Filesystem($adapter); + $this->LDViewService->setLDrawLibraryContext($this->ldrawLibraryContext); + } catch (Exception $exception) { + $this->logger->error($exception->getMessage()); + } + } + + public function loadOneModel($file) + { + $connection = $this->em->getConnection(); + try { + $connection->beginTransaction(); + + $this->loadModel($file); + + $connection->commit(); + } catch (\Exception $e) { + $connection->rollBack(); + $this->logger->error($e->getMessage()); + } + } + + public function loadAllModels() + { + $files = $this->finder->in([ + $this->ldrawLibraryContext->getAdapter()->getPathPrefix(), + ])->path('parts/')->name('*.dat')->depth(1)->files(); + + $this->initProgressBar($files->count()); + + /** @var SplFileInfo $file */ + foreach ($files as $file) { + $connection = $this->em->getConnection(); + $connection->beginTransaction(); + + try { + $this->progressBar->setMessage($file->getFilename(), 'filename'); + + $this->loadModel($file->getRealPath()); + + $connection->commit(); + } catch (\Exception $exception) { + $connection->rollBack(); + $this->logger->error($exception->getMessage()); + } + + $connection->close(); + + $this->progressBar->advance(); + } + $this->progressBar->finish(); + } + + /** + * Load Model entity into database. + * + * @param $file + * + * @return Model|null|false + */ + public function loadModel($file) + { + $fileContext = $this->getFileContext($file); + + // Return model from database if rewrite is not enabled + if (!$this->rewite && $model = $this->em->getRepository(Model::class)->findOneByNumber(basename($file, '.dat'))) { + return $model; + } + + // Parse model file save data to $modelArray + try { + $modelArray = $this->ldModelParser->parse($file); + } catch (ParseErrorException $e) { + $this->logger->error($e->getMessage(), [$file]); + + return null; + } catch (FileException $e) { + $this->logger->error($e->getMessage(), [$file]); + + return null; + } + + // Check if model fulfills rules and should be loaded + if ($this->isModelIncluded($modelArray)) { + // Recursively load model parent (if any) and add model id as alias of parent + if (($parentId = $this->getParentId($modelArray)) && ($parentModelFile = $this->findSubmodelFile($parentId, $fileContext)) !== null) { + $parentModel = $this->loadModel($parentModelFile); + + if ($parentModel) { + $alias = $this->em->getRepository(Alias::class)->getOrCreate($modelArray['id'], $parentModel); + $parentModel->addAlias($alias); + + $this->em->getRepository(Model::class)->save($parentModel); + } else { + $this->logger->info('Model skipped. ', ['number' => $modelArray['id'], 'parent' => $modelArray['parent']]); + } + + return $parentModel; + } + // Load model + + $model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']); + + // Recursively load models of subparts + if (isset($modelArray['subparts'])) { + foreach ($modelArray['subparts'] as $subpartId => $count) { + // Try to find model of subpart + if (($subpartFile = $this->findSubmodelFile($subpartId, $fileContext)) !== null) { + $subModel = $this->loadModel($subpartFile); + if ($subModel) { + $subpart = $this->em->getRepository(Subpart::class)->getOrCreate($model, $subModel, $count); + $model->addSubpart($subpart); + } + } else { + $this->logger->error('Subpart file not found', ['subpart' => $subpartId, 'model' => $modelArray]); + } + } + } + + // Add Keywords to model + if (isset($modelArray['keywords'])) { + foreach ($modelArray['keywords'] as $keyword) { + $keyword = $this->em->getRepository(Keyword::class)->getOrCreate(stripslashes(strtolower(trim($keyword)))); + $model->addKeyword($keyword); + } + } + + try { + // update model only if newer version + if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) { + $stl = $this->LDViewService->datToStl($file, $rewrite)->getPath(); + $model->setPath($stl); + + $model + ->setName($modelArray['name']) + ->setCategory($this->em->getRepository(Category::class)->getOrCreate($modelArray['category'])) + ->setAuthor($this->em->getRepository(Author::class)->getOrCreate($modelArray['author'])) + ->setModified($modelArray['modified']); + + $this->em->getRepository(Model::class)->save($model); + } + } catch (ConvertingFailedException $e) { + $this->logger->error($e->getMessage()); + + return null; + } + + return $model; + } + + return false; + } + + /** + * Get parent id of model from alias_model.yml if defined or return parent id loaded by LDModelParser. + * + * Used to eliminate duplicites of models in library, that could not be determined automatically + * + * @param $modelArray + * + * @return string + */ + private function getParentId($modelArray) + { + if ($this->relationMapper->find($modelArray['id'], 'alias_model') !== $modelArray['id']) { + return $this->relationMapper->find($modelArray['id'], 'alias_model'); + } + + return $modelArray['parent']; + } + + /** + * Find submodel file inside model context or inside LDraw library. + * + * + * LDraw.org Standards: File Format 1.0.2 (http://www.ldraw.org/article/218.html) + * + * "Sub-files can be located in the LDRAW\PARTS sub-directory, the LDRAW\P sub-directory, the LDRAW\MODELS sub-directory, + * the current file's directory, a path relative to one of these directories, or a full path may be specified. Sub-parts are typically + * stored in the LDRAW\PARTS\S sub-directory and so are referenced as s\subpart.dat, while hi-res primitives are stored in the + * LDRAW\P\48 sub-directory and so referenced as 48\hires.dat" + * + * + * @param $id + * @param Filesystem $context + * + * @return string + */ + private function findSubmodelFile($id, $context) + { + // Replace "\" directory separator used inside ldraw model files with system directoru separator + $filename = str_replace('\\', DIRECTORY_SEPARATOR, strtolower($id).'.dat'); + + // Try to find model in current file's directory + if ($context->has($filename)) { + return $context->getAdapter()->getPathPrefix().$filename; + } + // Try to find model in current LDRAW\PARTS sub-directory + elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) { + return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename; + } + // Try to find model in current LDRAW\P sub-directory + elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) { + return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename; + } + + return null; + } + + /** + * Get new filesystem context of current file. + * + * @param $file + * + * @return Filesystem + */ + private function getFileContext($file) + { + try { + $adapter = new Local(dirname($file)); + + return new Filesystem($adapter); + } catch (Exception $exception) { + $this->logger->error($exception->getMessage()); + } + } + + /** + * Determine if model file should be loaded into database. + * + * @param $modelArray + * + * @return bool + */ + private function isModelIncluded($modelArray) + { + // Do not include part primitives and subparts + if (in_array($modelArray['type'], ['48_Primitive', '8_Primitive', 'Primitive', 'Subpart'])) { + return false; + } + // Do not include sticker models + elseif ($modelArray['type'] == 'Sticker') { + $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'type' => $modelArray['type']]); + + return false; + } + // Do not include models without permission to redistribute + elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') { + $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]); + + return false; + } + + return true; + } +} diff --git a/src/AppBundle/Service/Loader/ModelLoaderService.php b/src/AppBundle/Service/Loader/ModelLoaderService.php deleted file mode 100644 index bbab54d..0000000 --- a/src/AppBundle/Service/Loader/ModelLoaderService.php +++ /dev/null @@ -1,250 +0,0 @@ -LDViewService = $LDViewService; - $this->ldrawService = $ldrawService; - $this->relationMapper = $relationMapper; - - $this->datParser = new DatParser(); - $this->finder = new Finder(); - } - - public function setLDrawLibraryContext($ldrawLibrary) - { - $adapter = new Local($ldrawLibrary); - $this->ldrawLibraryContext = new Filesystem($adapter); - - $this->LDViewService->setLdrawFilesystem($this->ldrawLibraryContext); - } - - public function setFileContext($file) { - $adapter = new Local($file); - $this->fileContext = new Filesystem($adapter); - } - - public function loadAllModels() - { - $files = $this->finder->in([$this->ldrawLibraryContext->getAdapter()->getPathPrefix()])->path('parts/')->name('*.dat')->depth(1)->files(); - - $modelManager = $this->ldrawService->getModelManager(); - - $this->initProgressBar($files->count()); - - /** @var SplFileInfo $file */ - foreach ($files as $file) { - $this->newModels = []; - - $model = $this->loadModel($file->getRealPath()); - - if($model !== null) { - $modelManager->getRepository()->save($model); - } - - $this->progressBar->advance(); - } - $this->progressBar->finish(); - } - - /** - * Load Model entity into database. - * - * @param $file - * - * @return Model|null - */ - public function loadModel($file) - { - $model = null; - - $modelManager = $this->ldrawService->getModelManager(); - $subpartManager = $this->ldrawService->getSubpartManager(); - - if(($model = $modelManager->findByNumber(basename($file,'.dat'))) || ($model = isset($this->newModels[basename($file,'.dat')]) ? $this->newModels[basename($file,'.dat')] : null)) { - return $model; - } - - try { - $modelArray = $this->datParser->parse($file); - } catch (\Exception $e) { - dump($e->getMessage()); - return null; - } - - if ($this->isModelIncluded($modelArray)) { - - if ($parentModelFile = $this->getParentModelFile($modelArray)) { - try { - if(($parentModel = $this->loadModel($parentModelFile))!= null) { - $alias = $this->ldrawService->getAliasManager()->create($modelArray['id'], $parentModel); - $parentModel->addAlias($alias); - - $this->newModels[$parentModel->getNumber()] = $parentModel; - return $parentModel; - } - } catch (\Exception $e) { - dump('b'); - dump($e->getMessage()); - return null; - } - } else { - $model = $modelManager->create($modelArray['id']); - - if (isset($modelArray['keywords'])) { - foreach ($modelArray['keywords'] as $keyword) { - $keyword = stripslashes(strtolower(trim($keyword))); - $model->addKeyword($this->ldrawService->getKeywordManager()->create($keyword)); - } - } - - if (isset($modelArray['subparts'])) { - foreach ($modelArray['subparts'] as $subpartId => $count) { - if(strpos($subpartId, 's\\') === false) { - if(($subpartFile = $this->findModelFile($subpartId)) != null) { - try { - if ($subModel = $this->loadModel($subpartFile)) { - $subpart = $subpartManager->create($model,$subModel,$count); - - $model->addSubpart($subpart); - } - } catch (\Exception $e) { - dump('c'); - dump($e->getMessage()); - } - } - } - } - } - - $model - ->setName($modelArray['name']) - ->setCategory($this->ldrawService->getCategoryManager()->create($modelArray['category'])) - ->setAuthor($modelArray['author']) - ->setModified($modelArray['modified']) - ->setPath($this->loadStlModel($file)); - - $this->LDViewService->datToPng($file); - - $this->newModels[$model->getNumber()] = $model; - } - } - - return $model; - } - - private function getParentModelFile($modelArray) { - if($this->relationMapper->find($modelArray['id'], 'alias_model') !== $modelArray['id']) { - return $this->findModelFile($this->relationMapper->find($modelArray['id'], 'alias_model')); - } else { - return strpos($modelArray['parent'], 's\\') === false ? $this->findModelFile($modelArray['parent']) : null; - } - } - - /** - * Find model file on ldraw filesystem. - * - * @param $id - * - * @return string - */ - private function findModelFile($id) - { - $filename = strtolower($id).'.dat'; - - if($this->fileContext && $this->fileContext->has($filename)) { - return $this->fileContext->getAdapter()->getPathPrefix().$filename; - } else if ($this->ldrawLibraryContext->has('parts/'.$filename)) { - return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts/'.$filename; - } - - return null; - } - - /** - * Determine if model file should be loaded into database. - * - * @param $modelArray - * - * @return bool - */ - private function isModelIncluded($modelArray) - { - // Do not include sticker parts and incomplete parts - if ( $modelArray['type'] != 'Subpart' && $modelArray['type'] != 'Sticker' ) { - return true; - } - - return false; - } - - /** - * Load stl model by calling LDViewSevice and create new Model. - * - * @param $file - * - * @throws \Exception - * - * @return string path of stl file - */ - private function loadStlModel($file) - { - try { - return $this->LDViewService->datToStl($file)->getPath(); - } catch (ConvertingFailedException $e) { - throw $e; //TODO - } - } -} diff --git a/src/AppBundle/Utils/DatParser.php b/src/AppBundle/Utils/LDModelParser.php similarity index 74% rename from src/AppBundle/Utils/DatParser.php rename to src/AppBundle/Utils/LDModelParser.php index 99f4b84..a789fc5 100644 --- a/src/AppBundle/Utils/DatParser.php +++ b/src/AppBundle/Utils/LDModelParser.php @@ -5,13 +5,11 @@ namespace AppBundle\Utils; use AppBundle\Exception\FileNotFoundException; use AppBundle\Exception\ParseErrorException; use League\Flysystem\File; -use Symfony\Component\Asset\Exception\LogicException; -class DatParser +class LDModelParser { /** - * Parse LDraw .dat file header identifying model store data to array. - * + * Parse LDraw model .dat file and return associative array in format: * [ * 'id' => string * 'name' => string @@ -20,17 +18,19 @@ class DatParser * 'author' => string * 'modified' => DateTime * 'type' => string - * 'subparts' => [] - * ] + * 'subparts' => [], + * 'licence' => string + * ]. * * LDraw.org Standards: Official Library Header Specification (http://www.ldraw.org/article/398.html) * - * @return array * @throws FileNotFoundException|ParseErrorException + * + * @return array */ public function parse($file) { - if(file_exists($file)) { + if (file_exists($file)) { $model = [ 'id' => null, 'name' => null, @@ -40,7 +40,8 @@ class DatParser 'modified' => null, 'type' => null, 'subparts' => [], - 'parent' => null + 'parent' => null, + 'license' => null, ]; try { @@ -63,19 +64,24 @@ class DatParser $model['name'] = preg_replace('/ {2,}/', ' ', ltrim($line, '=_')); $firstLine = true; - } // 0 !CATEGORY + } + // 0 !CATEGORY elseif (strpos($line, '!CATEGORY ') === 0) { $model['category'] = trim(preg_replace('/^!CATEGORY /', '', $line)); - } // 0 !KEYWORDS , , ..., + } + // 0 !KEYWORDS , , ..., elseif (strpos($line, '!KEYWORDS ') === 0) { $model['keywords'] = explode(', ', preg_replace('/^!KEYWORDS /', '', $line)); - } // 0 Name: .dat + } + // 0 Name: .dat elseif (strpos($line, 'Name: ') === 0 && !isset($header['id'])) { $model['id'] = preg_replace('/(^Name: )(.*)(.dat)/', '$2', $line); - } // 0 Author: [] + } + // 0 Author: [] elseif (strpos($line, 'Author: ') === 0) { $model['author'] = preg_replace('/^Author: /', '', $line); - } // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR + } + // 0 !LDRAW_ORG Part|Subpart|Primitive|48_Primitive|Shortcut (optional qualifier(s)) ORIGINAL|UPDATE YYYY-RR elseif (strpos($line, '!LDRAW_ORG ') === 0) { $type = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE| ORIGINAL)(.*)/', '$2', $line); @@ -84,17 +90,23 @@ class DatParser // Last modification date in format YYYY-RR $date = preg_replace('/(^!LDRAW_ORG )(.*)( UPDATE | ORIGINAL )(.*)/', '$4', $line); if (preg_match('/^[1-2][0-9]{3}-[0-9]{2}$/', $date)) { - $model['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date . '-01 00:00:00'); + $model['modified'] = \DateTime::createFromFormat('Y-m-d H:i:s', $date.'-01 00:00:00'); } } + // 0 !LICENSE Redistributable under CCAL version 2.0 : see CAreadme.txt | 0 !LICENSE Not redistributable : see NonCAreadme.txt + elseif (strpos($line, '!LICENSE ') === 0) { + $model['license'] = preg_replace('/(^!LICENSE )(.*) : (.*)$/', '$2', $line); + } } elseif (strpos($line, '1 ') === 0) { - $id = $this->getReferencedModelNumber($line); + $id = strtolower($this->getReferencedModelNumber($line)); - if(isset($model['subparts'][$id])) { + if (isset($model['subparts'][$id])) { $model['subparts'][$id] = $model['subparts'][$id] + 1; } else { $model['subparts'][$id] = 1; } + } elseif (!empty($line) && !in_array($line[0], ['2', '3', '4', '5'])) { + throw new ErrorParsingLineException($file,$line); } } @@ -102,14 +114,12 @@ class DatParser $model['type'] = 'Sticker'; } elseif (count($model['subparts']) == 1 && in_array($model['type'], ['Part Alias', 'Shortcut Physical_Colour', 'Shortcut Alias', 'Part Physical_Colour'])) { $model['parent'] = array_keys($model['subparts'])[0]; - } elseif ($parent = $this->getPrintedModelParentNumber($model['id'])) { + } elseif (($parent = $this->getPrintedModelParentNumber($model['id'])) && !in_array($model['type'], ['48_Primitive', '8_Primitive', 'Primitive', 'Subpart'])) { $model['type'] = 'Printed'; $model['parent'] = $parent; } elseif ($parent = $this->getObsoleteModelParentNumber($model['name'])) { $model['type'] = 'Alias'; $model['parent'] = $parent; - } elseif (strpos($model['name'], '~') === 0 && $model['type'] != 'Alias') { - $model['type'] = 'Obsolete/Subpart'; } fclose($handle); @@ -137,10 +147,10 @@ class DatParser */ public function getReferencedModelNumber($line) { - if(preg_match('/^1 16 0 0 0 -1 0 0 0 1 0 0 0 1 (.*)\.(dat|DAT)$/', $line, $matches)) - return null; - if (preg_match('/^1(.*) (.*)\.(dat|DAT)$/', $line, $matches)) { - return $matches[2]; + $line = ($line); + + if (preg_match('/^1(.*) (.*)\.dat$/', strtolower($line), $matches)) { + return str_replace('\\', DIRECTORY_SEPARATOR, $matches[2]); } return null; @@ -160,7 +170,7 @@ class DatParser */ public function getPrintedModelParentNumber($id) { - if (preg_match('/(^.*)(p[0-9a-z][0-9a-z][0-9a-z]{0,1})$/', $id, $matches)) { + if (preg_match('/(^.*)(p[0-9a-z]{2,3})$/', $id, $matches)) { return $matches[1]; } @@ -178,7 +188,7 @@ class DatParser * @param $name * @param $number * - * @return string|null LDraw number of printed part parent + * @return bool */ public function isSticker($name, $number) { @@ -186,8 +196,8 @@ class DatParser return true; } - // Check if in format nnnDaa == sticker - return preg_match('/(^.*)(d[a-z0-9][a-z0-9])$/', $number); + // Check if in format n*Daa == sticker + return preg_match('/(^.*)(d[0-9]{2})$/', $number); } /** @@ -205,7 +215,7 @@ class DatParser public function getObsoleteModelParentNumber($name) { if (preg_match('/^(~Moved to )(.*)$/', $name, $matches)) { - return $matches[2]; + return str_replace('\\', DIRECTORY_SEPARATOR, $matches[2]); } return null;