mirrors/synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2024-12-14 11:57:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Split setuping up and processing of tables

Browse source
This commit is contained in:
Erik Johnston 2015-04-27 17:53:40 +01:00
parent 40cbd6b6ee
commit 5b8b1a43bd
1 changed files with 23 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
scripts/port_from_sqlite_to_postgres.py
View file

@ -168,7 +168,7 @@ class Porter(object):
)
@defer.inlineCallbacks
def handle_table(self, table):
def setup_table(self, table):
def delete_all(txn):
txn.execute(
"DELETE FROM port_from_sqlite3 WHERE table_name = %s",
@ -287,6 +287,10 @@ class Porter(object):
postgres_size = yield self.postgres_store.execute(get_table_size)
defer.returnValue((table, postgres_size, table_size, next_chunk))
@defer.inlineCallbacks
def handle_table(self, table, postgres_size, table_size, next_chunk):
if not table_size:
return
@ -364,14 +368,14 @@ class Porter(object):
self.postgres_store = Store(postgres_db_pool, postgres_engine)
# Step 1. Set up databases.
self.progress.on_prepare_sqlite()
self.progress.set_state("Preparing SQLite3")
self.setup_db(sqlite_config, sqlite_engine)
self.progress.on_prepare_postgres()
self.progress.set_state("Preparing PostgreSQL")
self.setup_db(postgres_config, postgres_engine)
# Step 2. Get tables.
self.progress.fetching_tables()
self.progress.set_state("Fetching tables")
sqlite_tables = yield self.sqlite_store._simple_select_onecol(
table="sqlite_master",
keyvalues={
@ -390,7 +394,7 @@ class Porter(object):
tables = set(sqlite_tables) & set(postgres_tables)
self.progress.preparing_tables()
self.progress.set_state("Creating tables")
logger.info("Found %d tables", len(tables))
@ -409,10 +413,12 @@ class Porter(object):
except Exception as e:
logger.info("Failed to create port table: %s", e)
# Process tables.
yield defer.gatherResults(
self.progress.set_state("Preparing tables")
# Set up tables.
setup_res = yield defer.gatherResults(
[
self.handle_table(table)
self.setup_table(table)
for table in tables
if table not in ["schema_version", "applied_schema_deltas"]
and not table.startswith("sqlite_")
@ -420,6 +426,15 @@ class Porter(object):
consumeErrors=True,
)
# Process tables.
yield defer.gatherResults(
[
self.handle_table(*res)
for res in setup_res
],
consumeErrors=True,
)
self.progress.done()
except:
global end_error_exec_info