diff --git a/app/config/service/loader.yml b/app/config/service/loader.yml index 58a8ef8..9bd585d 100644 --- a/app/config/service/loader.yml +++ b/app/config/service/loader.yml @@ -3,11 +3,7 @@ services: abstract: true class: AppBundle\Service\Loader\BaseLoader calls: - - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']] - - service.ldview: - class: AppBundle\Service\LDViewService - arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem'] + - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader', '@app.transformer.format']] service.loader.rebrickable: class: AppBundle\Service\Loader\RebrickableLoader diff --git a/app/config/service/service.yml b/app/config/service/service.yml index 0013d14..7eb05a6 100644 --- a/app/config/service/service.yml +++ b/app/config/service/service.yml @@ -14,3 +14,7 @@ services: service.renderer.stl: class: AppBundle\Service\StlRendererService arguments: ['%kernel.root_dir%/Resources/povray_layout/layout.tmpl', '%povray_bin%','%stl2pov_bin%'] + + service.ldview: + class: AppBundle\Service\LDViewService + arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem'] diff --git a/src/AppBundle/Command/InitDataCommand.php b/src/AppBundle/Command/InitDataCommand.php new file mode 100644 index 0000000..2d2adc5 --- /dev/null +++ b/src/AppBundle/Command/InitDataCommand.php @@ -0,0 +1,67 @@ +setName('app:init') + ->setDescription('Loads relations between LDraw models and Rebrickable parts.') + ->setHelp('This command allows you to load relation between models and parts into database.') + ->setDefinition( + new InputDefinition([ + new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'), + ]) + ); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $loadModelsCommand = $this->getApplication()->find('app:load:models'); + + $returnCode = $loadModelsCommand->run(new ArrayInput([ + 'command' => 'app:load:models', + 'ldraw' => $input->getArgument('ldraw'), + '--all' => true + ]),$output); + + if($returnCode) { + return 1; + } + + $loadRebrickableCommad = $this->getApplication()->find('app:load:rebrickable'); + $returnCode = $loadRebrickableCommad->run(new ArrayInput([ + 'command' => 'app:load:rebrickable']),$output); + + if($returnCode) { + return 1; + } + + $loadRelationsCommand = $this->getApplication()->find('app:load:relations'); + + $returnCode = $loadRelationsCommand->run(new ArrayInput([ + 'command' => 'app:load:relations']),$output); + + if($returnCode) { + return 1; + } + + $loadImagesCommand = $this->getApplication()->find('app:load:images'); + + $returnCode = $loadImagesCommand->run(new ArrayInput([ + 'command' => 'app:load:images', + '--color' => -1, + '--rebrickable' => true, + '--models' => true + ]),$output); + } +} diff --git a/src/AppBundle/Command/LoadModelsCommand.php b/src/AppBundle/Command/LoadModelsCommand.php index c115440..54f925e 100644 --- a/src/AppBundle/Command/LoadModelsCommand.php +++ b/src/AppBundle/Command/LoadModelsCommand.php @@ -23,7 +23,6 @@ class LoadModelsCommand extends ContainerAwareCommand ->setDefinition( new InputDefinition([ new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'), -// new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'), new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'), new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'), new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'), @@ -35,8 +34,7 @@ class LoadModelsCommand extends ContainerAwareCommand { if (!$this->lock()) { $output->writeln('The command is already running in another process.'); - - return 0; + return 1; } $modelLoader = $this->getContainer()->get('service.loader.model'); @@ -51,12 +49,10 @@ class LoadModelsCommand extends ContainerAwareCommand if (($path = $input->getOption('file')) != null) { if ($file = realpath($path)) { $output->writeln([ - 'Loading model', - 'path: '.$file, - '------------------------------------------------------------------------------', + "Loading model: {$path}", ]); - $modelLoader->loadOneModel($file); + $modelLoader->loadOne($file); $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors(); $errors = $errorCount ? ''.$errorCount.'' : '0'; @@ -70,10 +66,12 @@ class LoadModelsCommand extends ContainerAwareCommand // Load all models inside ldraw/parts directory if ($input->getOption('all')) { $output->writeln([ - 'Loading models from LDraw library: '.$ldrawPath.'', + '------------------------------------------------------------------------------', + "Loading models from LDraw library: {$ldrawPath}", + '------------------------------------------------------------------------------', ]); - $modelLoader->loadAllModels(); + $modelLoader->loadAll(); $errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors(); $errors = $errorCount ? ''.$errorCount.'' : '0'; @@ -81,7 +79,9 @@ class LoadModelsCommand extends ContainerAwareCommand $output->writeln(['Done with "'.$errors.'" errors.']); } } else { - $output->writeln($ldraw.' is not valid path'); + $output->writeln("{$ldraw} is not a valid path!"); + $this->release(); + return 1; } $this->release(); diff --git a/src/AppBundle/Command/LoadRebrickableDataCommand.php b/src/AppBundle/Command/LoadRebrickableDataCommand.php index 71d7135..6ac5cb4 100644 --- a/src/AppBundle/Command/LoadRebrickableDataCommand.php +++ b/src/AppBundle/Command/LoadRebrickableDataCommand.php @@ -2,6 +2,7 @@ namespace AppBundle\Command; +use League\Flysystem\Exception; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -21,7 +22,11 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand $rebrickableLoader = $this->getContainer()->get('service.loader.rebrickable'); $rebrickableLoader->setOutput($output); - //TODO log errors - $rebrickableLoader->loadTables(); + try { + $rebrickableLoader->loadAll(); + } catch (Exception $exception) { + $output->writeln("{$exception->getMessage()}"); + return 1; + } } } diff --git a/src/AppBundle/Command/LoadRelationCommand.php b/src/AppBundle/Command/LoadRelationCommand.php index d90ebeb..1d18ffe 100644 --- a/src/AppBundle/Command/LoadRelationCommand.php +++ b/src/AppBundle/Command/LoadRelationCommand.php @@ -21,7 +21,15 @@ class LoadRelationCommand extends ContainerAwareCommand $relationLoader = $this->getContainer()->get('service.loader.relation'); $relationLoader->setOutput($output); - //TODO log errors - $relationLoader->loadNotPaired(); + + $output->writeln([ + '------------------------------------------------------------------------------', + "Loading relations between parts and models...", + '------------------------------------------------------------------------------', + ]); + + $relationLoader->loadAll(); + + $output->writeln(['Done!']); } } diff --git a/src/AppBundle/Service/Loader/BaseLoader.php b/src/AppBundle/Service/Loader/BaseLoader.php index 07343ee..8e222e2 100644 --- a/src/AppBundle/Service/Loader/BaseLoader.php +++ b/src/AppBundle/Service/Loader/BaseLoader.php @@ -4,6 +4,7 @@ namespace AppBundle\Service\Loader; use AppBundle\Exception\FileNotFoundException; use AppBundle\Exception\WriteErrorException; +use AppBundle\Transformer\FormatTransformer; use Doctrine\ORM\EntityManager; use Monolog\Logger; use Symfony\Bundle\FrameworkBundle\Translation\Translator; @@ -29,21 +30,29 @@ abstract class BaseLoader */ protected $progressBar; - /** @var Logger */ + /** + * @var Logger + */ protected $logger; + /** + * @var FormatTransformer + */ + private $formatTransformer; + /** * Loader constructor. * * @param EntityManager $em * @param Translator $translator */ - public function setArguments(EntityManager $em, $logger) + public function setArguments(EntityManager $em, $logger, $formatTransformer) { $this->em = $em; $this->em->getConnection()->getConfiguration()->setSQLLogger(null); $this->logger = $logger; + $this->formatTransformer = $formatTransformer; } public function setOutput(OutputInterface $output) @@ -57,28 +66,33 @@ abstract class BaseLoader * * @param $total */ - protected function initProgressBar($total) + protected function initProgressBar($total, $format = 'loader') { $this->progressBar = new ProgressBar($this->output, $total); -// $this->progressBar->setFormat('very_verbose'); - $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL); + ProgressBar::setFormatDefinition('loader', '[%current% / %max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%message%)' . PHP_EOL); + ProgressBar::setFormatDefinition('download', '[%progress% / %size%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)' . PHP_EOL); + $this->progressBar->setFormat($format); $this->progressBar->setBarWidth(70); + $this->progressBar->start(); } protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max) { + $this->logger->info($notification_code); + switch ($notification_code) { case STREAM_NOTIFY_FILE_SIZE_IS: $this->initProgressBar($bytes_max); - $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL); + $this->progressBar->setFormat('download'); + $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_max),'size'); + $this->progressBar->setRedrawFrequency($bytes_max/20); break; case STREAM_NOTIFY_PROGRESS: $this->progressBar->setProgress($bytes_transferred); + $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_transferred),'progress'); break; case STREAM_NOTIFY_COMPLETED: - $this->progressBar->setMessage('Done'); - $this->progressBar->setProgress($bytes_max); $this->progressBar->finish(); break; } @@ -106,6 +120,7 @@ abstract class BaseLoader if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) { throw new WriteErrorException($temp); } + $this->progressBar->finish(); } catch (ContextErrorException $e) { throw new FileNotFoundException($url); } catch (\Exception $e) { @@ -114,4 +129,9 @@ abstract class BaseLoader return $temp; } + + protected function writeOutput(array $lines) { + if($this->output) + $this->output->writeln($lines); + } } diff --git a/src/AppBundle/Service/Loader/ModelLoader.php b/src/AppBundle/Service/Loader/ModelLoader.php index 6e47507..64f08c8 100644 --- a/src/AppBundle/Service/Loader/ModelLoader.php +++ b/src/AppBundle/Service/Loader/ModelLoader.php @@ -12,15 +12,14 @@ use AppBundle\Exception\ConvertingFailedException; use AppBundle\Exception\FileException; use AppBundle\Exception\ParseErrorException; use AppBundle\Service\LDViewService; -use AppBundle\Utils\LDModelParser; -use AppBundle\Utils\RelationMapper; +use AppBundle\Util\LDModelParser; +use AppBundle\Util\RelationMapper; use League\Flysystem\Adapter\Local; use League\Flysystem\Exception; use League\Flysystem\Filesystem; use Symfony\Component\Finder\Finder; use Symfony\Component\Finder\SplFileInfo; -//TODO refactor class ModelLoader extends BaseLoader { /** @@ -80,7 +79,7 @@ class ModelLoader extends BaseLoader } } - public function loadOneModel($file) + public function loadOne($file) { $connection = $this->em->getConnection(); try { @@ -95,7 +94,7 @@ class ModelLoader extends BaseLoader } } - public function loadAllModels() + public function loadAll() { $files = $this->finder->in([ $this->ldrawLibraryContext->getAdapter()->getPathPrefix(), @@ -109,7 +108,7 @@ class ModelLoader extends BaseLoader $connection->beginTransaction(); try { - $this->progressBar->setMessage($file->getFilename(), 'filename'); + $this->progressBar->setMessage($file->getFilename()); $this->loadModel($file->getRealPath()); @@ -127,7 +126,8 @@ class ModelLoader extends BaseLoader } /** - * Load Model entity into database. + * Load model entity and all related submodels into database while generating stl file of model. + * Uses LDView to convert LDraw .dat to .stl * * @param $file * @@ -144,7 +144,8 @@ class ModelLoader extends BaseLoader // Parse model file save data to $modelArray try { - $modelArray = $this->ldModelParser->parse($file); + $content = file_get_contents($file); + $modelArray = $this->ldModelParser->parse($content); } catch (ParseErrorException $e) { $this->logger->error($e->getMessage(), [$file]); @@ -172,8 +173,8 @@ class ModelLoader extends BaseLoader return $parentModel; } - // Load model + // Load model $model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']); // Recursively load models of subparts @@ -203,7 +204,7 @@ class ModelLoader extends BaseLoader try { // update model only if newer version if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) { - $stl = $this->LDViewService->datToStl($file, $rewrite)->getPath(); + $stl = $this->LDViewService->datToStl($file, $this->rewite)->getPath(); $model->setPath($stl); $model @@ -271,12 +272,14 @@ class ModelLoader extends BaseLoader return $context->getAdapter()->getPathPrefix().$filename; } // Try to find model in current LDRAW\PARTS sub-directory - elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) { - return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename; - } - // Try to find model in current LDRAW\P sub-directory - elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) { - return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename; + elseif($this->ldrawLibraryContext) { + if($this->ldrawLibraryContext->has('parts/'.$filename)) { + return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename; + } + // Try to find model in current LDRAW\P sub-directory + elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) { + return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename; + } } return null; @@ -297,6 +300,7 @@ class ModelLoader extends BaseLoader return new Filesystem($adapter); } catch (Exception $exception) { $this->logger->error($exception->getMessage()); + return null; } } @@ -322,7 +326,6 @@ class ModelLoader extends BaseLoader // Do not include models without permission to redistribute elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') { $this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]); - return false; } diff --git a/src/AppBundle/Service/Loader/RebrickableLoader.php b/src/AppBundle/Service/Loader/RebrickableLoader.php index 7df7f91..ea68a1b 100644 --- a/src/AppBundle/Service/Loader/RebrickableLoader.php +++ b/src/AppBundle/Service/Loader/RebrickableLoader.php @@ -21,7 +21,7 @@ class RebrickableLoader extends BaseLoader $this->rebrickable_url = $rebrickable_url; } - public function loadTables() + public function loadAll() { $connection = $this->em->getConnection(); $connection->beginTransaction(); @@ -33,9 +33,8 @@ class RebrickableLoader extends BaseLoader $this->truncateTables(); $connection->prepare('SET foreign_key_checks = 1;')->execute(); - $this->output->writeln([ + $this->writeOutput([ 'Truncated rebrickable database tables.', - '------------------------------------------------------------------------------', 'Loading CSV files into database...', ]); @@ -54,7 +53,7 @@ class RebrickableLoader extends BaseLoader $connection->commit(); - $this->output->writeln('Rebrickable database loaded successfully!'); + $this->writeOutput(['Rebrickable database loaded successfully!']); } catch (\Exception $e) { $connection->rollBack(); throw $e; @@ -65,19 +64,17 @@ class RebrickableLoader extends BaseLoader { $array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors']; - $this->output->writeln([ - 'Loading Rebrickable CSV files', - '------------------------------------------------------------------------------', + $this->writeOutput([ + '------------------------------------------------------------------------------', + 'Loading Rebrickable CSV files', + '------------------------------------------------------------------------------', ]); foreach ($array as $item) { $this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv'); } - $this->output->writeln([ - 'Done', - '------------------------------------------------------------------------------', - ]); + $this->writeOutput(['All CSV files loaded.']); } private function truncateTables() diff --git a/src/AppBundle/Service/Loader/RelationLoader.php b/src/AppBundle/Service/Loader/RelationLoader.php index 7ceaf73..925fec7 100644 --- a/src/AppBundle/Service/Loader/RelationLoader.php +++ b/src/AppBundle/Service/Loader/RelationLoader.php @@ -30,26 +30,21 @@ class RelationLoader extends BaseLoader public function loadAll() { $parts = $this->em->getRepository(Part::class)->findAll(); - - $this->initProgressBar(count($parts)); - /** @var Part $part */ - foreach ($parts as $part) { - $this->load($part); - - $this->progressBar->advance(); - } - $this->progressBar->finish(); + $this->load($parts); } - public function loadNotPaired() + public function loadNotPaired($parts) { $parts = $this->em->getRepository(Part::class)->findAllNotPaired(); + $this->load($parts); + } + private function load($parts) { $this->initProgressBar(count($parts)); /** @var Part $part */ foreach ($parts as $part) { - $this->load($part); - $this->progressBar->setMessage($part->getNumber(), 'filename'); + $this->loadPartRelation($part); + $this->progressBar->setMessage($part->getNumber()); $this->progressBar->advance(); } $this->progressBar->finish(); @@ -62,7 +57,7 @@ class RelationLoader extends BaseLoader * * @return Model $m */ - private function load(Part $part) + private function loadPartRelation(Part $part) { $modelRepository = $this->em->getRepository(Model::class);