[PYTHON] [Django] Download large files while saving memory.

How to avoid memory errors when downloading large files with Django. Use StreamingHttpResponse and wsgiref.util.FileWrapper.

• Described an implementation example of function-based view

When using HttpResponse

from django.http import HttpResponse

def download_view(request, *args, **kwargs):
    with open('path/to/dir/small.csv', 'rb') as f:
        response = HttpResponse(f.read(), content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename=small.csv'
    return response

The entire capacity of the file is read into memory at the part of f.read ().

When using StreamingHttpResponse and FileWrapper

import os
from wsgiref.util import FileWrapper

from django.http import StreamingHttpResponse

#Read 8MB at a time
chunksize = 8 * (1024 ** 2)

def streaming_download_view(request, *args, **kwargs):
    path = os.path.join('path/to/dir', 'large.csv')
    response = StreamingHttpResponse(
        FileWrapper(open(path, 'rb'), chunksize),
        content_type='text/csv'
    )
    response['Content-Length'] = os.path.getsize(path)
    response['Content-Disposition'] = 'attachment; filename=large.csv'
    return response

Note that "filelike-object" is specified as the first argument of FileWrapper, but an error occurs (for some reason) when using the context manager.

~~ I haven't investigated much, but it's safe to use open () as in ~~ Document usage example.

Example of failure(Use context manager)

...
def streaming_download_view(request, *args, **kwargs):
    path = os.path.join('path/to/dir', 'large.csv')
    with open(path, 'rb') as f:
        response = StreamingHttpResponse(
            FileWrapper(f, chunksize),
            content_type='text/csv'
        )
        ...
        return response

Error output

Traceback (most recent call last):
  File "/home/ec2-user/.pyenv/versions/3.7.7/lib/python3.7/wsgiref/handlers.py", line 138, in run
    self.finish_response()
  File "/home/ec2-user/.pyenv/versions/3.7.7/lib/python3.7/wsgiref/handlers.py", line 183, in finish_response
    for data in self.result:
  File "/home/ec2-user/.pyenv/versions/3.7.7/lib/python3.7/wsgiref/util.py", line 30, in __next__
    data = self.filelike.read(self.blksize)
ValueError: read of closed file

Application: Delete file after download

If you want to treat the downloaded file as a temporary file, you can do it by combining it with Temporary Directory.

import os
from tempfile import TemporaryDirectory
from wsgiref.util import FileWrapper

from django.http import StreamingHttpResponse

#Read 8MB at a time
chunksize = 8 * (1024 ** 2)


def generate_return_file(target_dir):
    #Some process to create the file to be downloaded under the specified directory
    pass


def streaming_download_view_with_tempdir(request, *args, **kwargs):
    with TemporaryDirectory() as tempdir:
        path = os.path.join(tempdir, 'large.csv')
        #Create the target file under tempdir
        generate_return_file(tempdir)

        response = StreamingHttpResponse(
            FileWrapper(open(path, 'rb'), chunksize),
            content_type='text/csv'
        )
        response['Content-Length'] = os.path.getsize(path)
        response['Content-Disposition'] = 'attachment; filename=large.csv'
        return response