Doctrine 批量插入使用 2GB 内存

Question

我正在尝试通过 Doctrine Fixtures Bundle 使用 Symfony2 和 Doctrine 插入大约 200k 条记录。我正在使用 flush 和 clear，但最后脚本使用了 1.8 GB 的 RAM。

这是将 SmartMeter 实体加载到数据库中的 class：

<?php

namespace HTEC\SmartMeteringAPIBundle\DataFixtures\ORM;

use Doctrine\Common\DataFixtures\AbstractFixture;
use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;

use HTEC\SmartMeteringAPIBundle\Entity\SmartMeter;

use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;

class LoadSmartMeterData extends AbstractFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{

    static $NUMBER_OF_SMART_METERS = 0;
    static $MAX_NUM_OF_SM_PER_CONC = 500;

     /**
     * @var ContainerInterface
     */
    private $container;


    public function getOrder()
    {
        return 10; // the order in which fixtures will be loaded
    }

    /**
     * @inheritDoc
     */
    public function setContainer(ContainerInterface $container = null)
    {
        $this->container = $container;
    }

    /**
     * {@inheritDoc}
     */
    public function load(ObjectManager $manager)
    {

        $numberOfUsers = LoadUserData::$NUMBER_OF_USERS;
        $numberOfConcentrators = LoadConcentratorData::$NUMBER_OF_CONCENTRATORS;
        $numberOfTariffs = LoadTariffData::$NUMBER_OF_TARIFFS;

        $numberOfSmartMetersPerConcentrator = 0;

        $smartMeter = null;
        $concentrator = null;
        $user = null;
        $tariff = null;

        $feeders = array();
        $numberOfFeeders = 0;
        $feedersRandomKey = array();

        $isIpAddressDynamic = false;

        $numberOfTransformerFeeders = 0;

        $smartMeterType = 0;
        $smartMeterProtocol = 0;
        $modemType = 0;

        $numberOfSmartMeters = 0;


        echo "\n\nCreating Smart Meters. This could take a couple of minutes and could take approximately 2 GB of RAM \n\n...";

        $startTime = time();

        $smartMeterCount = 0;

        $lastChunkNumber = 0;
        $currentChunkNumber = 0;

        $lastSmartMeterNumber = 0;
        $smartMetersClearedCurrently = 0;

        $concentratorSmartMeters = null;

        for($i = 0; $i < LoadConcentratorData::$NUMBER_OF_CONCENTRATORS; $i++)
        {

            $concentrator = $manager->getRepository('SMAPIBundle:Concentrator')->find($i+1);

            $numberOfSmartMetersPerConcentrator = rand(1, self::$MAX_NUM_OF_SM_PER_CONC);

            $numberOfSmartMeters += $numberOfSmartMetersPerConcentrator;

            for($c = 0; $c < $numberOfSmartMetersPerConcentrator; $c++)
            {
                $smartMeter = new SmartMeter();
                $smartMeter->setSerialNumber(++$smartMeterCount);
                $smartMeter->setConcentrator($concentrator);

                $smartMeterType = rand(1,3);

                switch($smartMeterType)
                {
                    case 1:
                        $smartMeter->setType('DIRECT');
                    break;

                    case 2:
                        $smartMeter->setType('HALF');
                    break;

                    case 3:
                        $smartMeter->setType('INDIRECT');
                    break;
                }

                $user = $manager->getRepository('SMAPIBundle:User')->find(rand(1, $numberOfUsers));
                $smartMeter->setCreatedBy($user);

                $numberOfTransformerFeeders = $concentrator->getTransformerFeeders()->count();
                $feeders = $concentrator->getTransformerFeeders()->toArray();

                if($numberOfTransformerFeeders > 0)
                {

                    $feedersRandomKey = array_rand($feeders, 1);

                    if(isset($feeders[$feedersRandomKey]))
                    {
                        $smartMeter->setFeeder($feeders[$feedersRandomKey]);
                    }
                }

                $smartMeter->setStatus(rand(0,2));


                $tariff = $manager->getRepository('SMAPIBundle:Tariff')->find(rand(1, $numberOfTariffs));
                $smartMeter->setTariff($tariff);

                $smartMeterProtocol = rand(1,3);

                switch($smartMeterProtocol)
                {
                    case 1:
                        $smartMeter->setProtocol('DLMS');
                    break;

                    case 2:
                        $smartMeter->setProtocol('EURIDIS');
                    break;

                    case 3:
                        $smartMeter->setProtocol('IEC');
                    break;
                }


                $smartMeter->setModemSerialNumber(rand(1, 9000000000));

                $smartMeter->setManufacture('MAN: ' . ($i + $c));

                $modemType = rand(1,2);

                if($modemType === 1)
                {
                    $smartMeter->setModemType('PLC');
                    $smartMeter->setModemIndex(rand(1, 512));
                }
                else
                {

                    $smartMeter->setModemType('GPRS');
                    $isIpAddressDynamic = rand(0,1);

                    if($isIpAddressDynamic === 1)
                    {
                        $smartMeter->setModemIpAddress(rand(1000000, 2000000));
                    }
                    else
                    {
                        $smartMeter->setModemPhoneNumber($this->getRandomIpAddressV4());
                    }
                }


                if(rand(0,1) === 1)
                {
                    $smartMeter->setModemRepeaterNumber(rand(10000000, 90000000));
                }

                $manager->persist($smartMeter);

            } // end of FOR numberOfSmartMetersPerConcentrator

            // flush smart meters
            $manager->flush();
            $manager->clear();

            $currentChunkNumber = ceil($smartMeterCount / 5000);

            if($smartMeterCount > 5000 && $currentChunkNumber > $lastChunkNumber)
            {
                $lastChunkNumber = $currentChunkNumber;



                if($lastSmartMeterNumber > 0)
                {
                    $smartMetersClearedCurrently = $smartMeterCount - $lastSmartMeterNumber;
                }
                else
                {
                    $smartMetersClearedCurrently = $smartMeterCount;
                }


                echo "\n\nFlushing and clearing " . number_format($smartMetersClearedCurrently, 0, ',', '.') . " Smart Meters.\nTotal memory used after flush and clear: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

                $lastSmartMeterNumber = $smartMeterCount;
            }
        }// end for NUMBER_OF_CONCENTRATORS

        $manager->flush();
        $manager->clear();

        unset($concentrator);
        unset($smartMeter);
        unset($tariff);

        echo "\n\n-------------------------------------------\n\n";
        echo "\n\nTotal memory used after final flush and clear of Smart Meters: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

        echo "\n\nCreating Smart Meters complete. Created " . number_format($numberOfSmartMeters, 0, ',', '.') . " Smart Meters.\n\n";

        $durationSeconds = (time() - $startTime);
        $durationMinutes = $durationSeconds / 60;

        $secondsRemainder = $durationSeconds % 60;

        echo "\n\nTotal duration time: " . ceil($durationMinutes) . " minutes and " . $secondsRemainder . " seconds.\n\n\n";

        self::$NUMBER_OF_SMART_METERS = $numberOfSmartMeters;

    }

    public function getRandomIpAddressV4()
    {
        return rand(1, 255) . '.' . rand(0, 255) . '.' . rand(0, 255) . '.' . rand(0, 255);
    }
}

在大约 5000 条记录上，我调用了 flush 和 clear，但似乎没有释放任何内存。

是否有关于如何防止 Doctrine 在批处理任务期间使用过多 RAM 的建议？

Answer 1

如果它是一个命令，尝试运行它并带有选项 --no-debug。或者，您可以通过在 load 函数的开头调用 $manager->getConnection()->getConfiguration()->setSQLLogger(null); 来禁用记录器。

在 Doctrine 批处理任务期间，无论哪种方式禁用记录器都会节省大量内存。

Doctrine 批量插入使用 2GB 内存

Doctrine batch inserting uses 2GB of Ram

php

doctrine

symfony

batch-insert

doctrine-orm