v/PrintABrick
1
0
Fork 0
mirror of https://github.com/ToxicCrack/PrintABrick.git synced 2025-05-19 13:40:08 -07:00
Code Issues Projects Releases Wiki Activity

Download rebrickable csv with command

Browse Source
This commit is contained in:
David Hübner 2017-01-09 14:26:45 +01:00
parent c696185b0a
commit 863c931593
2 changed files with 105 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/AppBundle/Command/LoadRebrickableCommand.php
View File

@ -3,7 +3,6 @@
namespace AppBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
@ -14,10 +13,7 @@ class LoadRebrickableCommand extends ContainerAwareCommand
$this
->setName('app:loadRebrickable')
->setDescription('Loads Rebrickable csv data')
->setHelp("This command allows you to..")
->addArgument('pieces', InputArgument::REQUIRED, 'Path to Rebrickable pieces.csv file')
->addArgument('sets', InputArgument::REQUIRED, 'Path to Rebrickable sets.csv file')
->addArgument('set_pieces', InputArgument::REQUIRED, 'Path to Rebrickable set_pieces.csv file')
->setHelp('This command allows you to..')
;
}
@ -25,13 +21,16 @@ class LoadRebrickableCommand extends ContainerAwareCommand
{
$modelLoader = $this->getContainer()->get('app.model_loader_service');
printf('colors'."\n");
try {
$modelLoader->loadColors();
printf('sets'."\n");
$modelLoader->loadBuildingKits(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('sets'));
printf('pieces'."\n");
$modelLoader->loadParts(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('pieces'));
printf('set_pieces'."\n");
$modelLoader->loadPartBuildingKits(getcwd().DIRECTORY_SEPARATOR.$input->getArgument('set_pieces'));
$modelLoader->loadParts($output);
$modelLoader->loadBuildingKits($output);
$modelLoader->loadPartBuildingKits($output);
} catch (\Exception $e) {
printf($e->getMessage());
}
}
}

113
src/AppBundle/Service/ModelLoaderService.php
View File

@ -12,6 +12,7 @@ use AppBundle\Entity\Part;
use AppBundle\Entity\Part_BuildingKit;
use Doctrine\ORM\EntityManager;
use Symfony\Component\Asset\Exception\LogicException;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Finder\Finder;
use Symfony\Component\Finder\SplFileInfo;
@ -92,26 +93,30 @@ class ModelLoaderService
// Rebrickable
public function loadPartBuildingKits($set_pieces) {
public function loadPartBuildingKits($output)
{
$partRepository = $this->em->getRepository('AppBundle:Part');
$buldingKitRepository = $this->em->getRepository('AppBundle:BuildingKit');
$colorRepository = $this->em->getRepository('AppBundle:Color');
$setPieces = tempnam(sys_get_temp_dir(), 'printabrick.');
file_put_contents($setPieces, fopen('compress.zlib://http://rebrickable.com/files/set_pieces.csv.gz', 'r'));
$this->em->getConnection()->getConfiguration()->setSQLLogger(null);
if (($handle = fopen($set_pieces, "r")) !== FALSE) {
$header = fgetcsv($handle, 200, ",");
$line = 0;
while (($data = fgetcsv($handle, 200, ",")) !== FALSE) {
if (($handle = fopen($setPieces, 'r')) !== false) {
$header = fgetcsv($handle, 200, ',');
// create a new progress bar (50 units)
$progress = new ProgressBar($output, intval(exec("wc -l '$setPieces'")));
$progress->setFormat('very_verbose');
$progress->setBarWidth(50);
$progress->start();
$index = 0;
while (($data = fgetcsv($handle, 200, ',')) !== false) {
$partBuildingKit = new Part_BuildingKit();
$line = $line + 1;
if($line%1000 == 0 ) {
var_dump($line);
$this->em->flush();
}
$buildingKit = $buldingKitRepository->findOneBy(['number' => $data[0]]);
$part = $partRepository->findOneBy(['number' => $data[1]]);
$color = $colorRepository->findOneBy(['id' => $data[3]]);
@ -126,23 +131,46 @@ class ModelLoaderService
$this->em->persist($partBuildingKit);
}
$index = $index + 1;
if ($index % 25 == 0) {
$this->em->flush();
$this->em->clear();
}
$progress->advance();
}
$this->em->flush();
$this->em->clear();
fclose($handle);
}
$progress->finish();
$progress->clear();
}
unlink($setPieces);
}
public function loadBuildingKits($sets_file) {
public function loadBuildingKits($output)
{
$keywordRepository = $this->em->getRepository('AppBundle:Keyword');
if (($handle = fopen($sets_file, "r")) !== FALSE) {
$header = fgetcsv($handle, 1000, ",");
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$sets = tempnam(sys_get_temp_dir(), 'printabrick.');
file_put_contents($sets, fopen('compress.zlib://http://rebrickable.com/files/sets.csv.gz', 'r'));
if (($handle = fopen($sets, 'r')) !== false) {
$header = fgetcsv($handle, 500, ',');
// create a new progress bar (50 units)
$progress = new ProgressBar($output, intval(exec("wc -l '$sets'")));
$progress->setFormat('very_verbose');
$progress->setBarWidth(50);
$progress->start();
$index = 0;
while (($data = fgetcsv($handle, 500, ',')) !== false) {
$buildingKit = new BuildingKit();
for($i=3; $i <=5; $i++) {
for ($i = 3; $i <= 5; ++$i) {
$keyword = $keywordRepository->findOneBy(['name' => $data[$i]]);
if ($keyword == null) {
$keyword = new Keyword();
@ -154,7 +182,6 @@ class ModelLoaderService
$buildingKit->addKeyword($keyword);
}
$buildingKit
->setNumber($data[0])
->setYear($data[1])
@ -163,23 +190,45 @@ class ModelLoaderService
$this->em->persist($buildingKit);
$index = $index + 1;
if ($index % 25 == 0) {
$this->em->flush();
$this->em->clear();
}
$progress->advance();
}
$this->em->flush();
$this->em->clear();
fclose($handle);
$progress->finish();
$progress->clear();
}
unlink($sets);
}
public function loadParts($pieces_file) {
public function loadParts($output)
{
$pieces = tempnam(sys_get_temp_dir(), 'printabrick.');
file_put_contents($pieces, fopen('compress.zlib://http://rebrickable.com/files/pieces.csv.gz', 'r'));
$categoryRepository = $this->em->getRepository('AppBundle:Category');
if (($handle = fopen($pieces_file, "r")) !== FALSE) {
$header = fgetcsv($handle, 300, ",");
while (($data = fgetcsv($handle, 300, ",")) !== FALSE) {
if (($handle = fopen($pieces, 'r')) !== false) {
// create a new progress bar (50 units)
$progress = new ProgressBar($output, intval(exec("wc -l '$pieces'")));
$progress->setFormat('very_verbose');
$progress->setBarWidth(50);
$progress->start();
$header = fgetcsv($handle, 300, ',');
while (($data = fgetcsv($handle, 300, ',')) !== false) {
$part = new Part();
$part->setNumber($data[0])->setName($data[1]);
$category = $categoryRepository->findOneBy(['name' => $data[2]]);
if ($category == null) {
$category = new Category();
@ -194,16 +243,23 @@ class ModelLoaderService
$this->em->persist($part);
// $this->em->persist($category);
$progress->advance();
}
$this->em->flush();
$this->em->clear();
fclose($handle);
}
$progress->finish();
$progress->clear();
}
public function loadColors() {
unlink($pieces);
}
public function loadColors()
{
$rb_colors = $this->rebrickableManager->getColors();
foreach ($rb_colors as $rb_color) {
@ -219,7 +275,8 @@ class ModelLoaderService
$this->em->flush();
}
public function getModel(Part $part) {
public function getModel(Part $part)
{
$modelRepository = $this->em->getRepository('AppBundle:Model');
if (strpos($part->getNumber(), 'p')) {