Change storage layout of projects.

A project is now completely contained in a directory, the metadata file moved into the directory as 'project.noise'.
startup
Ben Niemann 2019-06-09 17:19:34 +02:00
parent eba3e1abbf
commit dec25ac24c
6 changed files with 38 additions and 51 deletions

View File

@ -52,7 +52,7 @@ class CorruptedProjectError(Error):
pass
HeaderData = TypedDict('HeaderData', {'data_dir': str, 'created': int})
HeaderData = TypedDict('HeaderData', {'created': int})
LogEntry = bytes
Checkpoint = bytes
@ -81,7 +81,6 @@ class ProjectStorage(object):
def __init__(self) -> None:
self.path = None # type: str
self.data_dir = None # type: str
self.header_data = None # type: HeaderData
self.file_lock = None # type: IO
self.log_index_fp = None # type: IO[bytes]
@ -118,7 +117,7 @@ class ProjectStorage(object):
logger.info("Opening project at %s", self.path)
try:
fp = fileutil.File(path)
fp = fileutil.File(os.path.join(path, 'project.noise'))
file_info, self.header_data = fp.read_json()
except fileutil.Error as exc:
raise FileOpenError(str(exc))
@ -129,16 +128,10 @@ class ProjectStorage(object):
if file_info.version not in self.SUPPORTED_VERSIONS:
raise UnsupportedFileVersionError()
self.data_dir = os.path.join(
os.path.dirname(self.path), self.header_data['data_dir'])
if not os.path.isdir(self.data_dir):
raise CorruptedProjectError(
"Directory %s missing" % self.data_dir)
self.file_lock = self.acquire_file_lock(
os.path.join(self.data_dir, "lock"))
os.path.join(self.path, "lock"))
log_path = os.path.join(self.data_dir, 'log.%06d' % self.log_file_number)
log_path = os.path.join(self.path, 'log.%06d' % self.log_file_number)
if os.path.exists(log_path):
mode = 'a+b'
else:
@ -146,7 +139,7 @@ class ProjectStorage(object):
self.log_fp = open(log_path, mode=mode, buffering=0)
self.log_index_fp = open(
os.path.join(self.data_dir, 'log.index'),
os.path.join(self.path, 'log.index'),
mode='r+b', buffering=0)
self.log_index = bytearray(self.log_index_fp.read())
self.next_log_number = len(self.log_index) // self.log_index_formatter.size
@ -155,7 +148,7 @@ class ProjectStorage(object):
self.written_log_number = self.next_log_number - 1
self.log_history_fp = open(
os.path.join(self.data_dir, 'log.history'),
os.path.join(self.path, 'log.history'),
mode='r+b', buffering=0)
self.log_history = bytearray(self.log_history_fp.read())
self.next_sequence_number = len(self.log_history) // self.log_history_formatter.size
@ -171,7 +164,7 @@ class ProjectStorage(object):
self.redo_count = 0
self.checkpoint_index_fp = open(
os.path.join(self.data_dir, 'checkpoint.index'),
os.path.join(self.path, 'checkpoint.index'),
mode='r+b', buffering=0)
self.checkpoint_index = bytearray(self.checkpoint_index_fp.read())
self.next_checkpoint_number = (
@ -199,17 +192,15 @@ class ProjectStorage(object):
def create(cls, path: str) -> 'ProjectStorage':
header_data = {
'created': int(time.time()),
'data_dir': os.path.splitext(os.path.basename(path))[0] + '.data',
} # type: HeaderData
data_dir = os.path.join(os.path.dirname(path), header_data['data_dir'])
os.mkdir(data_dir)
os.mkdir(path)
for fname in ('lock', 'log.index', 'log.history',
'checkpoint.index'):
open(os.path.join(data_dir, fname), 'wb').close()
open(os.path.join(path, fname), 'wb').close()
fp = fileutil.File(path)
fp = fileutil.File(os.path.join(path, 'project.noise'))
fp.write_json(
header_data,
fileutil.FileInfo(
@ -299,7 +290,7 @@ class ProjectStorage(object):
def _write_checkpoint(
self, seq_number: int, checkpoint_number: int, checkpoint: Checkpoint) -> None:
checkpoint_path = os.path.join(
self.data_dir,
self.path,
'checkpoint.%06d' % checkpoint_number)
logger.info("Writing checkpoint %s...", checkpoint_path)
with open(checkpoint_path, mode='wb', buffering=0) as fp:
@ -359,7 +350,7 @@ class ProjectStorage(object):
except KeyError:
log_fp = open(
os.path.join(
self.data_dir,
self.path,
'log.%06d' % file_number),
mode='r+b', buffering=0)
self.log_fp_map[self.log_file_number] = log_fp
@ -505,7 +496,7 @@ class ProjectStorage(object):
checkpoint_number = self._get_checkpoint_entry(checkpoint_number)[1]
checkpoint_path = os.path.join(
self.data_dir,
self.path,
'checkpoint.%06d' % checkpoint_number)
logger.info("Reading checkpoint %s...", checkpoint_path)
with open(checkpoint_path, mode='rb') as fp:

View File

@ -123,17 +123,19 @@ class StorageTest(unittest.TestCase):
ps.close()
self.assertTrue(
self.fake_os.path.isfile('/foo.data/log.index'))
self.fake_os.path.isfile('/foo/project.noise'))
self.assertTrue(
self.fake_os.path.isfile('/foo/log.index'))
self.assertEqual(
self.fake_os.path.getsize('/foo.data/log.index'),
self.fake_os.path.getsize('/foo/log.index'),
3 * ps.log_index_formatter.size)
self.assertTrue(
self.fake_os.path.isfile('/foo.data/log.history'))
self.fake_os.path.isfile('/foo/log.history'))
self.assertEqual(
self.fake_os.path.getsize('/foo.data/log.history'),
self.fake_os.path.getsize('/foo/log.history'),
7 * ps.log_history_formatter.size)
self.assertTrue(
self.fake_os.path.isfile('/foo.data/log.000000'))
self.fake_os.path.isfile('/foo/log.000000'))
def test_undo_the_undone(self):
ps = storage.ProjectStorage.create('/foo')
@ -235,11 +237,11 @@ class StorageTest(unittest.TestCase):
ps.close()
self.assertTrue(
self.fake_os.path.isfile('/foo.data/checkpoint.index'))
self.fake_os.path.isfile('/foo/checkpoint.index'))
self.assertEqual(
self.fake_os.path.getsize('/foo.data/checkpoint.index'),
self.fake_os.path.getsize('/foo/checkpoint.index'),
2 * ps.checkpoint_index_formatter.size)
self.assertTrue(
self.fake_os.path.isfile('/foo.data/checkpoint.000000'))
self.fake_os.path.isfile('/foo/checkpoint.000000'))
self.assertTrue(
self.fake_os.path.isfile('/foo.data/checkpoint.000001'))
self.fake_os.path.isfile('/foo/checkpoint.000001'))

View File

@ -112,16 +112,16 @@ class ProjectTest(
async def test_create(self):
p = await project.Project.create_blank(
path='/foo.noise',
path='/foo',
pool=self.pool,
writer=self.writer_client,
node_db=self.node_db)
await p.close()
self.assertTrue(self.fake_os.path.isfile('/foo.noise'))
self.assertTrue(self.fake_os.path.isdir('/foo.data'))
self.assertTrue(self.fake_os.path.isdir('/foo'))
self.assertTrue(self.fake_os.path.isfile('/foo/project.noise'))
f = fileutil.File('/foo.noise')
f = fileutil.File('/foo/project.noise')
file_info, contents = f.read_json()
self.assertEqual(file_info.version, 1)
self.assertEqual(file_info.filetype, 'project-header')
@ -155,7 +155,7 @@ class ProjectTest(
async def test_create_checkpoint(self):
p = await project.Project.create_blank(
path='/foo.noise',
path='/foo',
pool=self.pool,
writer=self.writer_client,
node_db=self.node_db)
@ -165,7 +165,7 @@ class ProjectTest(
await p.close()
self.assertTrue(
self.fake_os.path.isfile('/foo.data/checkpoint.000001'))
self.fake_os.path.isfile('/foo/checkpoint.000001'))
async def test_merge_mutations(self):
p = await project.Project.create_blank(

View File

@ -86,7 +86,7 @@ class WriterProcess(core.ProcessBase):
assert self.__storage is None
self.__storage = storage.ProjectStorage.create(request.path)
response.data_dir = self.__storage.data_dir
response.data_dir = self.__storage.path
self.__storage.add_checkpoint(request.initial_checkpoint)
response.storage_state.CopyFrom(self.__get_storage_state())
@ -100,7 +100,7 @@ class WriterProcess(core.ProcessBase):
self.__storage = storage.ProjectStorage()
self.__storage.open(request.path)
response.data_dir = self.__storage.data_dir
response.data_dir = self.__storage.path
checkpoint_number, actions = self.__storage.get_restore_info()

View File

@ -277,7 +277,7 @@ class NewProjectDialog(ui_base.CommonMixin, QtWidgets.QDialog):
return self.__name.text()
def projectPath(self) -> str:
filename = self.projectName() + '.noise'
filename = self.projectName()
filename = filename.replace('%', '%25')
filename = filename.replace('/', '%2F')
return os.path.join(self.projectDir(), filename)

View File

@ -125,11 +125,7 @@ class Project(ui_base.CommonMixin, Item):
self.client = None
async def delete(self) -> None:
os.unlink(self.path)
data_dir = os.path.join(
os.path.dirname(self.path),
os.path.basename(self.path)[:-6] + '.data')
shutil.rmtree(data_dir)
shutil.rmtree(self.path)
class ProjectRegistry(ui_base.CommonMixin, QtCore.QAbstractItemModel):
@ -153,12 +149,10 @@ class ProjectRegistry(ui_base.CommonMixin, QtCore.QAbstractItemModel):
directory = os.path.abspath(directory)
logger.info("Scanning project directory %s...", directory)
for dirpath, dirnames, filenames in os.walk(directory):
for filename in filenames:
if filename.endswith('.noise'):
data_dir_name = filename[:-6] + '.data'
if data_dir_name in dirnames:
dirnames.remove(data_dir_name)
paths.append(os.path.join(dirpath, filename))
for dirname in list(dirnames):
if os.path.isfile(os.path.join(dirpath, dirname, 'project.noise')):
dirnames.remove(dirname)
paths.append(os.path.join(dirpath, dirname))
return paths