Skip to content

Instantly share code, notes, and snippets.

@abaybek

abaybek/bench_po.py

Last active September 6, 2019 06:25
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save abaybek/24614e66e7e9cf3e33f03c07f543aa76 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save abaybek/24614e66e7e9cf3e33f03c07f543aa76 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. abaybek revised this gist Sep 6, 2019. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions bench_po.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    # file: core/management/commands/bench_po.py

    from timeit import default_timer as timer
    from memory_profiler import profile
    from django.core.management.base import BaseCommand
  2. abaybek renamed this gist Sep 6, 2019. 1 changed file with 0 additions and 0 deletions.
    0 .py → bench_po.py
    View file
    File renamed without changes.
  3. abaybek created this gist Sep 6, 2019.
    53 changes: 53 additions & 0 deletions .py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,53 @@
    from timeit import default_timer as timer
    from memory_profiler import profile
    from django.core.management.base import BaseCommand
    from core.models import CoreData


    class Command(BaseCommand):
    def add_arguments(self, parser):
    parser.add_argument('batch_size', type=int)

    @profile
    def iterate_all(self):
    qs = CoreData.objects.all()
    counter = 0
    for counter, o in enumerate(qs.iterator(chunk_size=self.batch_size)):
    o
    print(f'Found {counter}')

    @profile
    def get_all_batch(self):
    qs = CoreData.objects.all()
    counter = 0
    for counter, o in enumerate(qs):
    o
    print(f'Found {counter}')

    @profile
    def get_chanked_batch(self):
    qs = CoreData.objects.all()
    counter = 0

    def batch_qs(qs, batch_size=1000):
    total = qs.count()
    for start in range(0, total, batch_size):
    end = min(start + batch_size, total)
    yield qs[start:end]

    for batch in batch_qs(qs, batch_size=self.batch_size):
    for document in batch:
    document
    counter += 1

    print(f'Found {counter}')

    def handle(self, *args, **options):
    self.batch_size = options['batch_size']
    start = timer()
    self.iterate_all()
    print(timer() - start)


    # self.get_all_batch()
    # self.get_chanked_batch()