v/PrintABrick
1
0
Fork 0
mirror of https://github.com/ToxicCrack/PrintABrick.git synced 2025-05-21 06:30:10 -07:00
Code Issues Projects Releases Wiki Activity

Improve load rebrickable data commad

Browse Source
This commit is contained in:
David Hübner 2017-04-16 22:18:05 +02:00
parent 8a1a5e2cff
commit 809b86ce19
4 changed files with 170 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/config/config.yml
View File

@ -7,8 +7,8 @@ imports:
# http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
parameters:
locale: en
# rebrickable csv files root URL
rebrickable_url: 'http://rebrickable.com/media/downloads/'
# rebrickable csv files root URL (http://rebrickable.com/media/downloads/ or local dir containing csv files)
rebrickable_csv_url: 'http://rebrickable.com/media/downloads/'
framework:
#esi: ~

4
src/AppBundle/Command/LoadRebrickableDataCommand.php
View File

@ -12,8 +12,8 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
{
$this
->setName('app:load:rebrickable')
->setDescription('Loads Rebrickable database')
->setHelp('This command allows you to..');
->setDescription('Loads Rebrickable data about sets and parts into database.')
->setHelp('This command allows you to load Rebrickable CSV files containing information about sets and parts into database.');
}
protected function execute(InputInterface $input, OutputInterface $output)

166
src/AppBundle/Service/Loader/RebrickableLoader.php Normal file
View File

@ -0,0 +1,166 @@
<?php
namespace AppBundle\Service\Loader;
use AppBundle\Entity\Rebrickable\Part;
use AppBundle\Entity\Rebrickable\Set;
use Doctrine\ORM\Query\Expr\Join;
//TODO Refactor + validate csv files
class RebrickableLoader extends BaseLoader
{
private $rebrickable_url;
private $csvFile;
/**
* ModelLoaderService constructor.
*/
public function __construct($rebrickable_url)
{
$this->rebrickable_url = $rebrickable_url;
}
public function loadTables()
{
$connection = $this->em->getConnection();
$connection->beginTransaction();
try {
$this->loadCsvFiles();
$connection->prepare('SET foreign_key_checks = 0;')->execute();
$this->truncateTables();
$connection->prepare('SET foreign_key_checks = 1;')->execute();
$this->output->writeln([
'<info>Truncated</info> <comment>rebrickable</comment> <info>database tables.</info>',
'------------------------------------------------------------------------------',
'Loading CSV files into database...',
]);
$this->loadColorTable($this->csvFile['colors']);
$this->loadCategoryTable($this->csvFile['part_categories']);
$this->loadPartTable($this->csvFile['parts']);
$this->loadThemeTable($this->csvFile['themes']);
$this->loadSetTable($this->csvFile['sets']);
$this->loadInventoryTable($this->csvFile['inventories']);
$this->loadInventorySetTable($this->csvFile['inventory_sets']);
$connection->prepare('SET foreign_key_checks = 0;')->execute();
$this->loadInventoryPartTable($this->csvFile['inventory_parts']);
$connection->prepare('SET foreign_key_checks = 1;')->execute();
$this->addMissingParts();
$connection->commit();
$this->output->writeln('Rebrickable database loaded successfully!');
} catch (\Exception $e) {
$connection->rollBack();
throw $e;
}
}
private function loadCSVFiles()
{
$array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
$this->output->writeln([
'Loading Rebrickable CSV files',
'------------------------------------------------------------------------------',
]);
foreach ($array as $item) {
$this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
}
$this->output->writeln([
'<info>Done</info>',
'------------------------------------------------------------------------------',
]);
}
private function truncateTables()
{
$query = '
TRUNCATE TABLE rebrickable_inventory_parts;
TRUNCATE TABLE rebrickable_inventory_sets;
TRUNCATE TABLE rebrickable_inventory;
TRUNCATE TABLE rebrickable_set;
TRUNCATE TABLE rebrickable_theme;
TRUNCATE TABLE rebrickable_part;
TRUNCATE TABLE rebrickable_category;
TRUNCATE TABLE rebrickable_color;
';
return $this->em->getConnection()->prepare($query)->execute();
}
private function loadCsvFile($file, $table, $columns)
{
$query = sprintf("LOAD DATA LOCAL INFILE '%s'
REPLACE INTO TABLE %s
FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
LINES TERMINATED BY '\\n'
IGNORE 1 LINES %s", addslashes($file), $table, $columns);
return $this->em->getConnection()->prepare($query)->execute();
}
private function addMissingParts()
{
$connection = $this->em->getConnection();
$statement = $connection->prepare(
'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
WHERE rebrickable_part.id IS NULL');
$statement->execute();
$foreignKeys = $statement->fetchAll();
foreach ($foreignKeys as $foreignKey) {
$part = new Part();
$part->setNumber($foreignKey['part_id']);
$this->em->getRepository(Part::class)->save($part);
}
}
private function loadInventoryTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
}
private function loadInventoryPartTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
}
private function loadInventorySetTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
}
private function loadSetTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
}
private function loadThemeTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
}
private function loadPartTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_part', '(`id`,`name`,`category_id`)');
}
private function loadCategoryTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_category', '(`id`,`name`)');
}
private function loadColorTable($csv)
{
return $this->loadCsvFile($csv, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
}
}

152
src/AppBundle/Service/Loader/RebrickableLoaderService.php
View File

@ -1,152 +0,0 @@
<?php
namespace AppBundle\Service\Loader;
use AppBundle\Entity\Rebrickable\Inventory_Part;
use AppBundle\Entity\Rebrickable\Inventory_Set;
use AppBundle\Entity\Rebrickable\Part;
use AppBundle\Entity\Rebrickable\Set;
use Doctrine\ORM\Query\Expr\Join;
//TODO Refactor + validate csv files
class RebrickableLoaderService extends BaseLoaderService
{
private $rebrickable_url;
/**
* ModelLoaderService constructor.
*/
public function __construct($rebrickable_url)
{
$this->rebrickable_url = $rebrickable_url;
}
public function loadTables()
{
$connection = $this->em->getConnection();
try {
$connection->beginTransaction();
$connection->prepare('SET foreign_key_checks = 0;')->execute();
$this->truncateTables();
$connection->prepare('SET foreign_key_checks = 1;')->execute();
$this->loadColorTable();
$this->loadCategoryTable();
$this->loadPartTable();
$this->loadThemeTable();
$this->loadSetTable();
$this->loadInventoryTable();
$this->loadInventorySetTable();
$connection->prepare('SET foreign_key_checks = 0;')->execute();
$this->loadInventoryPartTable();
$connection->prepare('SET foreign_key_checks = 1;')->execute();
$this->addMissingParts();
$connection->commit();
} catch (\Exception $e) {
$connection->rollBack();
throw $e;
}
}
private function truncateTables()
{
$query ='
TRUNCATE TABLE rebrickable_inventory_parts;
TRUNCATE TABLE rebrickable_inventory_sets;
TRUNCATE TABLE rebrickable_color;
TRUNCATE TABLE rebrickable_category;
TRUNCATE TABLE rebrickable_inventory;
TRUNCATE TABLE rebrickable_set;
TRUNCATE TABLE rebrickable_theme;
TRUNCATE TABLE rebrickable_part;
';
return $this->em->getConnection()->prepare($query)->execute();
}
private function loadCsvFile($file, $table, $columns)
{
$query = sprintf("LOAD DATA LOCAL INFILE '%s'
REPLACE INTO TABLE %s
FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
LINES TERMINATED BY '\\n'
IGNORE 1 LINES %s", addslashes($file), $table, $columns);
return $this->em->getConnection()->prepare($query)->execute();
}
private function addMissingParts() {
$connection = $this->em->getConnection();
$statement = $connection->prepare(
'SELECT DISTINCT rebrickable_inventory_parts.part_id FROM rebrickable_inventory_parts
LEFT JOIN rebrickable_part ON rebrickable_inventory_parts.part_id = rebrickable_part.id
WHERE rebrickable_part.id IS NULL');
$statement->execute();
$foreignKeys = $statement->fetchAll();
foreach ($foreignKeys as $foreignKey) {
$part = new Part();
$part->setNumber($foreignKey['part_id']);
$this->em->getRepository(Part::class)->save($part);
}
}
private function loadInventoryTable()
{
$file = $this->downloadFile($this->rebrickable_url.'inventories.csv');
return $this->loadCsvFile($file, 'rebrickable_inventory', '(`id`,`version`,`set_id`)');
}
private function loadInventoryPartTable()
{
$file = $this->downloadFile($this->rebrickable_url.'inventory_parts.csv');
return $this->loadCsvFile($file, 'rebrickable_inventory_parts', '(`inventory_id`,`part_id`,`color_id`,`quantity`, @var) SET spare = IF(@var=\'t\',1,0)');
}
private function loadInventorySetTable()
{
$file = $this->downloadFile($this->rebrickable_url.'inventory_sets.csv');
return $this->loadCsvFile($file, 'rebrickable_inventory_sets', '(`inventory_id`,`set_id`,`quantity`)');
}
private function loadSetTable()
{
$file = $this->downloadFile($this->rebrickable_url.'sets.csv');
return $this->loadCsvFile($file, 'rebrickable_set', '(`id`,`name`,`year`,`theme_id`,`num_parts`)');
}
private function loadThemeTable()
{
$file = $this->downloadFile($this->rebrickable_url.'themes.csv');
return $this->loadCsvFile($file, 'rebrickable_theme', '(`id`,`name`,@var) SET parent_id = nullif(@var,\'\')');
}
private function loadPartTable()
{
$file = $this->downloadFile($this->rebrickable_url.'parts.csv');
return $this->loadCsvFile($file, 'rebrickable_part', '(`id`,`name`,`category_id`)');
}
private function loadCategoryTable()
{
$file = $this->downloadFile($this->rebrickable_url.'part_categories.csv');
return $this->loadCsvFile($file, 'rebrickable_category', '(`id`,`name`)');
}
private function loadColorTable()
{
$file = $this->downloadFile($this->rebrickable_url.'colors.csv');
return $this->loadCsvFile($file, 'rebrickable_color', '(`id`,`name`,`rgb`, @var) SET transparent = IF(@var=\'t\',1,0)');
}
}