mirror of
https://github.com/ytdl-org/youtube-dl.git
synced 2024-09-12 00:29:29 -04:00
[cache] Add cache validation by program version, based on yt-dlp
This commit is contained in:
parent
55c823634d
commit
218c423bc0
@ -3,17 +3,18 @@
|
|||||||
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
|
||||||
from test.helper import FakeYDL
|
from test.helper import FakeYDL
|
||||||
from youtube_dl.cache import Cache
|
from youtube_dl.cache import Cache
|
||||||
|
from youtube_dl.utils import version_tuple
|
||||||
|
from youtube_dl.version import __version__
|
||||||
|
|
||||||
|
|
||||||
def _is_empty(d):
|
def _is_empty(d):
|
||||||
@ -54,6 +55,17 @@ class TestCache(unittest.TestCase):
|
|||||||
self.assertFalse(os.path.exists(self.test_dir))
|
self.assertFalse(os.path.exists(self.test_dir))
|
||||||
self.assertEqual(c.load('test_cache', 'k.'), None)
|
self.assertEqual(c.load('test_cache', 'k.'), None)
|
||||||
|
|
||||||
|
def test_cache_validation(self):
|
||||||
|
ydl = FakeYDL({
|
||||||
|
'cachedir': self.test_dir,
|
||||||
|
})
|
||||||
|
c = Cache(ydl)
|
||||||
|
obj = {'x': 1, 'y': ['ä', '\\a', True]}
|
||||||
|
c.store('test_cache', 'k.', obj)
|
||||||
|
self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
|
||||||
|
new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
|
||||||
|
self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
@ -10,12 +10,21 @@ import traceback
|
|||||||
|
|
||||||
from .compat import compat_getenv
|
from .compat import compat_getenv
|
||||||
from .utils import (
|
from .utils import (
|
||||||
|
error_to_compat_str,
|
||||||
expand_path,
|
expand_path,
|
||||||
|
is_outdated_version,
|
||||||
|
try_get,
|
||||||
write_json_file,
|
write_json_file,
|
||||||
)
|
)
|
||||||
|
from .version import __version__
|
||||||
|
|
||||||
|
|
||||||
class Cache(object):
|
class Cache(object):
|
||||||
|
|
||||||
|
_YTDL_DIR = 'youtube-dl'
|
||||||
|
_VERSION_KEY = _YTDL_DIR + '_version'
|
||||||
|
_DEFAULT_VERSION = '2021.12.17'
|
||||||
|
|
||||||
def __init__(self, ydl):
|
def __init__(self, ydl):
|
||||||
self._ydl = ydl
|
self._ydl = ydl
|
||||||
|
|
||||||
@ -23,7 +32,7 @@ class Cache(object):
|
|||||||
res = self._ydl.params.get('cachedir')
|
res = self._ydl.params.get('cachedir')
|
||||||
if res is None:
|
if res is None:
|
||||||
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
||||||
res = os.path.join(cache_root, 'youtube-dl')
|
res = os.path.join(cache_root, self._YTDL_DIR)
|
||||||
return expand_path(res)
|
return expand_path(res)
|
||||||
|
|
||||||
def _get_cache_fn(self, section, key, dtype):
|
def _get_cache_fn(self, section, key, dtype):
|
||||||
@ -50,13 +59,22 @@ class Cache(object):
|
|||||||
except OSError as ose:
|
except OSError as ose:
|
||||||
if ose.errno != errno.EEXIST:
|
if ose.errno != errno.EEXIST:
|
||||||
raise
|
raise
|
||||||
write_json_file(data, fn)
|
write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
|
||||||
except Exception:
|
except Exception:
|
||||||
tb = traceback.format_exc()
|
tb = traceback.format_exc()
|
||||||
self._ydl.report_warning(
|
self._ydl.report_warning(
|
||||||
'Writing cache to %r failed: %s' % (fn, tb))
|
'Writing cache to %r failed: %s' % (fn, tb))
|
||||||
|
|
||||||
def load(self, section, key, dtype='json', default=None):
|
def _validate(self, data, min_ver):
|
||||||
|
version = try_get(data, lambda x: x[self._VERSION_KEY])
|
||||||
|
if not version: # Backward compatibility
|
||||||
|
data, version = {'data': data}, self._DEFAULT_VERSION
|
||||||
|
if not is_outdated_version(version, min_ver or '0', assume_new=False):
|
||||||
|
return data['data']
|
||||||
|
self._ydl.to_screen(
|
||||||
|
'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
|
||||||
|
|
||||||
|
def load(self, section, key, dtype='json', default=None, min_ver=None):
|
||||||
assert dtype in ('json',)
|
assert dtype in ('json',)
|
||||||
|
|
||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
@ -66,12 +84,12 @@ class Cache(object):
|
|||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
|
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
|
||||||
return json.load(cachef)
|
return self._validate(json.load(cachef), min_ver)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
try:
|
try:
|
||||||
file_size = os.path.getsize(cache_fn)
|
file_size = os.path.getsize(cache_fn)
|
||||||
except (OSError, IOError) as oe:
|
except (OSError, IOError) as oe:
|
||||||
file_size = str(oe)
|
file_size = error_to_compat_str(oe)
|
||||||
self._ydl.report_warning(
|
self._ydl.report_warning(
|
||||||
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||||
except IOError:
|
except IOError:
|
||||||
|
Loading…
Reference in New Issue
Block a user