Django file storage (2) custom storage system, django file storage

Django file storage (2) custom storage system, django file storage

To write a storage system by yourself, follow these steps:

1. Write a file inherited from django. core. files. storage. Storage.

from django.core.files.storage import Storageclass MyStorage(Storage):    ...

2. Django must be able to instantiate MyStorage without any parameters, so any environment settings must come from django. conf. settings.

from django.conf import settingsfrom django.core.files.storage import Storageclass MyStorage(Storage):    def __init__(self, option=None):        if not option:            option = settings.CUSTOM_STORAGE_OPTIONS        ...

3. source code based on the open and save methods of Storage:

    def open(self, name, mode='rb'):        """        Retrieves the specified file from storage.        """        return self._open(name, mode)    def save(self, name, content, max_length=None):        """        Saves new content to the file specified by name. The content should be        a proper File object or any python file-like object, ready to be read        from the beginning.        """        # Get the proper name for the file, as it will actually be saved.        if name is None:            name = content.name        if not hasattr(content, 'chunks'):            content = File(content, name)        name = self.get_available_name(name, max_length=max_length)        return self._save(name, content)

MyStorage needs to implement the _ open and _ save methods.

If you are writing a local storage system, you must rewrite the path method.

4. Use django. utils. deconstruct. deconstructible to enable serialization in migration.

Also, the Storage. delete (), Storage. exists (), Storage. listdir (), Storage. size (), and Storage. url () methods all report NotImplementedError, which also needs to be rewritten.

Django Qiniu Storage

Qiniu cloud has its own django storage system. You can see how it works at https://github.com/glasslion/django-qiniu-storage.

Configure QINIU_ACCESS_KEY, QINIU_SECRET_KEY, QINIU_BUCKET_NAME, QINIU_BUCKET_DOMAIN, and QINIU_SECURE_URL in the environment variable or settings.

Use qiniu cloud to host files uploaded by users, and set DEFAULT_FILE_STORAGE in settings. py:

DEFAULT_FILE_STORAGE = 'qiniustorage.backends.QiniuStorage'

Use qiniu to host dynamically generated files and static files of the site, and set:

STATICFILES_STORAGE  = 'qiniustorage.backends.QiniuStaticStorage'

Run python manage. py collectstatic and the static files will be uniformly uploaded to qiniu.

The QiniuStorage code is as follows:

@deconstructibleclass QiniuStorage(Storage):    """    Qiniu Storage Service    """    location = ""    def __init__(            self,            access_key=QINIU_ACCESS_KEY,            secret_key=QINIU_SECRET_KEY,            bucket_name=QINIU_BUCKET_NAME,            bucket_domain=QINIU_BUCKET_DOMAIN,            secure_url=QINIU_SECURE_URL):        self.auth = Auth(access_key, secret_key)        self.bucket_name = bucket_name        self.bucket_domain = bucket_domain        self.bucket_manager = BucketManager(self.auth)        self.secure_url = secure_url    def _clean_name(self, name):        """        Cleans the name so that Windows style paths work        """        # Normalize Windows style paths        clean_name = posixpath.normpath(name).replace('\\', '/')        # os.path.normpath() can strip trailing slashes so we implement        # a workaround here.        if name.endswith('/') and not clean_name.endswith('/'):            # Add a trailing slash as it was stripped.            return clean_name + '/'        else:            return clean_name    def _normalize_name(self, name):        """        Normalizes the name so that paths like /path/to/ignored/../foo.txt        work. We check to make sure that the path pointed to is not outside        the directory specified by the LOCATION setting.        """        base_path = force_text(self.location)        base_path = base_path.rstrip('/')        final_path = urljoin(base_path.rstrip('/') + "/", name)        base_path_len = len(base_path)        if (not final_path.startswith(base_path) or                final_path[base_path_len:base_path_len + 1] not in ('', '/')):            raise SuspiciousOperation("Attempted access to '%s' denied." %                                      name)        return final_path.lstrip('/')    def _open(self, name, mode='rb'):        return QiniuFile(name, self, mode)    def _save(self, name, content):        cleaned_name = self._clean_name(name)        name = self._normalize_name(cleaned_name)        if hasattr(content, 'chunks'):            content_str = b''.join(chunk for chunk in content.chunks())        else:            content_str = content.read()        self._put_file(name, content_str)        return cleaned_name    def _put_file(self, name, content):        token = self.auth.upload_token(self.bucket_name)        ret, info = put_data(token, name, content)        if ret is None or ret['key'] != name:            raise QiniuError(info)    def _read(self, name):        return requests.get(self.url(name)).content    def delete(self, name):        name = self._normalize_name(self._clean_name(name))        if six.PY2:            name = name.encode('utf-8')        ret, info = self.bucket_manager.delete(self.bucket_name, name)        if ret is None or info.status_code == 612:            raise QiniuError(info)    def _file_stat(self, name, silent=False):        name = self._normalize_name(self._clean_name(name))        if six.PY2:            name = name.encode('utf-8')        ret, info = self.bucket_manager.stat(self.bucket_name, name)        if ret is None and not silent:            raise QiniuError(info)        return ret    def exists(self, name):        stats = self._file_stat(name, silent=True)        return True if stats else False    def size(self, name):        stats = self._file_stat(name)        return stats['fsize']    def modified_time(self, name):        stats = self._file_stat(name)        time_stamp = float(stats['putTime']) / 10000000        return datetime.datetime.fromtimestamp(time_stamp)    def listdir(self, name):        name = self._normalize_name(self._clean_name(name))        if name and not name.endswith('/'):            name += '/'        dirlist = bucket_lister(self.bucket_manager, self.bucket_name,                                prefix=name)        files = []        dirs = set()        base_parts = name.split("/")[:-1]        for item in dirlist:            parts = item['key'].split("/")            parts = parts[len(base_parts):]            if len(parts) == 1:                # File                files.append(parts[0])            elif len(parts) > 1:                # Directory                dirs.add(parts[0])        return list(dirs), files    def url(self, name):        name = self._normalize_name(self._clean_name(name))        name = filepath_to_uri(name)        protocol = u'https://' if self.secure_url else u'http://'        return urljoin(protocol + self.bucket_domain, name)

The configuration is obtained from environment variables or settings. py:

def get_qiniu_config(name, default=None):    """    Get configuration variable from environment variable    or django setting.py    """    config = os.environ.get(name, getattr(settings, name, default))    if config is not None:        if isinstance(config, six.string_types):            return config.strip()        else:            return config    else:        raise ImproperlyConfigured(            "Can't find config for '%s' either in environment"            "variable or in setting.py" % name)QINIU_ACCESS_KEY = get_qiniu_config('QINIU_ACCESS_KEY')QINIU_SECRET_KEY = get_qiniu_config('QINIU_SECRET_KEY')QINIU_BUCKET_NAME = get_qiniu_config('QINIU_BUCKET_NAME')QINIU_BUCKET_DOMAIN = get_qiniu_config('QINIU_BUCKET_DOMAIN', '').rstrip('/')QINIU_SECURE_URL = get_qiniu_config('QINIU_SECURE_URL', 'False')

The _ open and _ save methods are rewritten:

    def _open(self, name, mode='rb'):        return QiniuFile(name, self, mode)    def _save(self, name, content):        cleaned_name = self._clean_name(name)        name = self._normalize_name(cleaned_name)        if hasattr(content, 'chunks'):            content_str = b''.join(chunk for chunk in content.chunks())        else:            content_str = content.read()        self._put_file(name, content_str)        return cleaned_name

Use the put_data method to upload files. The related code is as follows:

Def put_data (up_token, key, data, params = None, mime_type = 'application/octet-stream', check_crc = False, progress_handler = None, fname = None ): "" Upload binary stream to qiniu Args: up_token: Upload credential key: Upload File Name data: Upload binary stream params: Custom variable, specification reference http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar mime_type: Upload data mimeType check_crc: check crc32 progress_handler: Upload progress Returns: A dict variable, similar to {"hash": "<Hash string>", "key ": "<Key string>"} A ResponseInfo object "crc = crc32 (data) if check_crc else None return _ form_put (up_token, key, data, params, mime_type, crc, progress_handler, fname) def _ form_put (up_token, key, data, params, mime_type, crc, progress_handler = None, file_name = None): fields = {} if params: for k, v in params. items (): fields [k] = str (v) if crc: fields ['crc32'] = crc if key is not None: fields ['key'] = key fields ['Token'] = up_token url = config. get_default ('default _ zone '). get_up_host_by_token (up_token) + '/' # name = key if key else file_name fname = file_name if not fname or not fname. strip (): fname = 'file _ name' r, info = http. _ post_file (url, data = fields, files = {'file': (fname, data, mime_type)}) if r is None and info. need_retry (): if info. connect_failed: url = config. get_default ('default _ zone '). get_up_host_backup_by_token (up_token) + '/' if hasattr (data, 'read') is False: pass elif hasattr (data, 'seek ') and (not hasattr (data, 'seekable') or data. seekable (): data. seek (0) else: return r, info r, info = http. _ post_file (url, data = fields, files = {'file': (fname, data, mime_type)}) return r, infodef _ post_file (url, data, files ): return _ post (url, data, files, None) def _ post (url, data, files, auth, headers = None): if _ session is None: _ init () try: post_headers = _ headers. copy () if headers is not None: for k, v in headers. items (): post_headers.update ({k: v}) r = _ session. post (url, data = data, files = files, auth = auth, headers = post_headers, timeout = config. get_default ('Connection _ timeout') failed t Exception as e: return None, ResponseInfo (None, e) return _ return_wrapper (r) def _ init (): session = requests. session () adapter = requests. adapters. HTTPAdapter (pool_connections = config. get_default ('Connection _ pool '), pool_maxsize = config. get_default ('Connection _ pool '), max_retries = config. get_default ('Connection _ retries ') session. mount ('HTTP: // ', adapter) global _ session = session

Finally, the requests library is used to upload files, which is applicable to the number of connection pools and the number of link retries.