The procedure editor database is SQL Server. Jianhan provides a dump of this data, which is converted to Postgres using DBBeaver.
This page lists the Postgres queries necessary to produce CSV files to populate Data Graphs.
Sourced from:
https://docs.google.com/spreadsheets/d/1e3AnQebAO5ug-Pc_0qDq9KkyZiy0dRhJMvm0lRRJOXk/
Sourced from:
https://docs.google.com/spreadsheets/d/1e3AnQebAO5ug-Pc_0qDq9KkyZiy0dRhJMvm0lRRJOXk/
COPY (
SELECT *
FROM procedure.layingbody
)
TO '/Users/smethurstm/Documents/procedure-editor/organisations-accountable-parliament.csv' DELIMITER ',' CSV HEADER;
Populated by hand (open / closed).
COPY (
SELECT *
FROM procedure.proceduresteppublication
)
TO '/Users/smethurstm/Documents/procedure-editor/publications.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.legislature
)
TO '/Users/smethurstm/Documents/procedure-editor/legislatures.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.house
)
TO '/Users/smethurstm/Documents/procedure-editor/houses.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.proceduresteptype
)
TO '/Users/smethurstm/Documents/procedure-editor/step-types.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.solractofparliamentdata
)
TO '/Users/smethurstm/Documents/procedure-editor/acts-of-parliament.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.procedurecalculationstyle
)
TO '/Users/smethurstm/Documents/procedure-editor/calculation-styles.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.procedurestep
WHERE proceduresteptypeid != 1
)
TO '/Users/smethurstm/Documents/procedure-editor/non-business-steps.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT p.*, calculation_styles.calculation_styles_string
FROM procedure.procedure p
LEFT JOIN
(
SELECT pcsa.procedureid AS procedure_id, STRING_AGG(pcs.id::text, ', ') AS calculation_styles_string
FROM procedure.procedurecalculationstyleapplicability pcsa, procedure.procedurecalculationstyle pcs
WHERE pcsa.procedurecalculationstyleid = pcs.id
GROUP BY procedure_id
) calculation_styles
ON calculation_styles.procedure_id = p.id
)
TO '/Users/smethurstm/Documents/procedure-editor/procedures.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.procedurestepcollection
)
TO '/Users/smethurstm/Documents/procedure-editor/step-collections.csv' DELIMITER ',' CSV HEADER;
Populated by hand.
COPY (
SELECT
s.*,
CASE
WHEN step_houses.houses_string = 'House of Commons' OR step_houses.houses_string = 'House of Lords' OR step_houses.houses_string = 'House of Commons and House of Lords'
THEN CONCAT( s.procedurestepname, ' (', step_houses.houses_string, ')' )
WHEN legislature.name = 'Scottish Parliament' OR legislature.name = 'Senedd Cymru' OR legislature.name = 'Northern Ireland Assembly'
THEN CONCAT( s.procedurestepname, ' (', legislature.name, ')' )
ELSE
s.procedurestepname
END AS label,
collection_memberships.step_collections_concatenated AS step_collections_concatenated,
step_houses.step_houses_concatenated AS step_houses_concatenated,
actualised_alongsides.actualised_alongside_concatenated AS actualised_alongside_concatenated
FROM procedure.procedurestep s
LEFT JOIN
(
SELECT scm.procedurestepid AS step_id, STRING_AGG(sc.id::text, ', ') AS step_collections_concatenated
FROM procedure.procedurestepcollectionmembership scm, procedure.procedurestepcollection sc
WHERE scm.procedurestepcollectionid = sc.id
GROUP BY step_id
) collection_memberships
ON collection_memberships.step_id = s.id
LEFT JOIN
(
SELECT sh.procedurestepid AS step_id, STRING_AGG(h.id::text, ', ') AS step_houses_concatenated, STRING_AGG(h.housename::text, ' and ') AS houses_string
FROM procedure.procedurestephouse sh, procedure.house h
WHERE sh.houseid = h.id
GROUP BY step_id
) step_houses
ON step_houses.step_id = s.id
LEFT JOIN
(
SELECT sas.procedurestepid AS from_step_id, STRING_AGG(sas.commonlyactualisedalongsideprocedurestepid::text, ', ') AS actualised_alongside_concatenated
FROM procedure.procedurestepalongsidestep sas
GROUP BY from_step_id
) actualised_alongsides
ON actualised_alongsides.from_step_id = s.id
LEFT JOIN
(
SELECT l.id, l.legislaturename AS name
FROM procedure.legislature l
) legislature
ON legislature.id = s.legislatureid
WHERE s.proceduresteptypeid = 1
)
TO '/Users/smethurstm/Documents/procedure-editor/business_steps.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.procedurestepdisplaydepth
)
TO '/Users/smethurstm/Documents/procedure-editor/step-display-depths.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT *
FROM procedure.procedureclock
)
TO '/Users/smethurstm/Documents/procedure-editor/clocks.csv' DELIMITER ',' CSV HEADER;
We need to flag routes forming part of a non-component procedure as being not required for export, where those routes also form part of a component procedure.
We add a is_included_in_export
boolean to the procedurerouteprocedure join table.
ALTER TABLE procedure.procedurerouteprocedure
ADD is_included_in_export BOOLEAN DEFAULT TRUE;
We set the is_included_in_export
boolean to false for any route forming part of a non-component procedure, where that route also forms part of a component procedure.
This update flags routes forming part of the EVEL component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 11
)
AND procedure.procedurerouteprocedure.procedureid != 11;
This update flags routes forming part of the SLSC component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 8
)
AND procedure.procedurerouteprocedure.procedureid != 8;
This update flags routes forming part of the JCSI component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 28
)
AND procedure.procedurerouteprocedure.procedureid != 28;
This update flags routes forming part of the SCSI component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 9
)
AND procedure.procedurerouteprocedure.procedureid != 9;
This update flags routes forming part of the Commons first reading component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 41
)
AND procedure.procedurerouteprocedure.procedureid != 41;
This update flags routes forming part of the Commons ten minute rule bill component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 44
)
AND procedure.procedurerouteprocedure.procedureid != 44;
This update flags routes forming part of the treaty BAT component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 61
)
AND procedure.procedurerouteprocedure.procedureid != 61;
This update flags routes forming part of the treaty EFRA component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 29
)
AND procedure.procedurerouteprocedure.procedureid != 29;
This update flags routes forming part of the treaty HAC component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 66
)
AND procedure.procedurerouteprocedure.procedureid != 66;
This update flags routes forming part of the treaty IAC and EAC component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 16
)
AND procedure.procedurerouteprocedure.procedureid != 16;
This update flags routes forming part of the treaty ITC component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 26
)
AND procedure.procedurerouteprocedure.procedureid != 26;
This update flags routes forming part of the treaty JCHR component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 15
)
AND procedure.procedurerouteprocedure.procedureid != 15;
This update flags routes forming part of the treaty Lords’ committees component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 13
)
AND procedure.procedurerouteprocedure.procedureid != 13;
This update flags routes forming part of the treaty NIAC component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 56
)
AND procedure.procedurerouteprocedure.procedureid != 56;
This update flags routes forming part of the treaty other Commons’ committees component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 65
)
AND procedure.procedurerouteprocedure.procedureid != 65;
This update flags routes forming part of the treaty pre-laying component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 62
)
AND procedure.procedurerouteprocedure.procedureid != 62;
This update flags routes forming part of the DPRRC scrutiny reserve component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 60
)
AND procedure.procedurerouteprocedure.procedureid != 60;
This update flags routes forming part of the Great Steward of Scotland component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 40
)
AND procedure.procedurerouteprocedure.procedureid != 40;
This update flags routes forming part of the legislative consent motion Northern Ireland component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 58
)
AND procedure.procedurerouteprocedure.procedureid != 58;
This update flags routes forming part of the legislative consent motion Scotland component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 46
)
AND procedure.procedurerouteprocedure.procedureid != 46;
This update flags routes forming part of the legislative consent motion Welsh component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 57
)
AND procedure.procedurerouteprocedure.procedureid != 57;
This update flags routes forming part of the LRO Commons committee consideration component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 53
)
AND procedure.procedurerouteprocedure.procedureid != 53;
This update flags routes forming part of the LRO Commons procedure determination component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 51
)
AND procedure.procedurerouteprocedure.procedureid != 51;
This update flags routes forming part of the LRO draft affirmative component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 49
)
AND procedure.procedurerouteprocedure.procedureid != 49;
This update flags routes forming part of the LRO draft negative component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 48
)
AND procedure.procedurerouteprocedure.procedureid != 48;
This update flags routes forming part of the LRO Lords’ Committee consideration component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 52
)
AND procedure.procedurerouteprocedure.procedureid != 52;
This update flags routes forming part of the LRO Lords’ debates component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 55
)
AND procedure.procedurerouteprocedure.procedureid != 55;
This update flags routes forming part of the LRO Lords’ procedure determination component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 50
)
AND procedure.procedurerouteprocedure.procedureid != 50;
This update flags routes forming part of the local government finance report component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 12
)
AND procedure.procedurerouteprocedure.procedureid != 12;
This update flags routes forming part of the Prince of Wales’ consent component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 39
)
AND procedure.procedurerouteprocedure.procedureid != 39;
This update flags routes forming part of the Queens’ consent component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 37
)
AND procedure.procedurerouteprocedure.procedureid != 37;
This update flags routes forming part of the PNSI Commons’ committee component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 77
)
AND procedure.procedurerouteprocedure.procedureid != 77;
This update flags routes forming part of the scrutiny reserve JCHR component procedure, where those routes form part of a different procedure:
UPDATE procedure.procedurerouteprocedure SET is_included_in_export = FALSE
WHERE procedure.procedurerouteprocedure.procedurerouteid IN (
SELECT rp.procedurerouteid
FROM procedure.procedurerouteprocedure rp
WHERE rp.procedureid = 76
)
AND procedure.procedurerouteprocedure.procedureid != 76;
A query for Jayne to check that routes flagged as not included for export look correct.
COPY (
SELECT p.procedurename AS procedure, from_s.procedurestepname AS from_step, to_s.procedurestepname AS to_step
FROM procedure.procedurerouteprocedure pr, procedure.procedure p, procedure.procedureroute r, procedure.procedurestep from_s, procedure.procedurestep to_s
WHERE pr.is_included_in_export is FALSE
AND pr.procedureid = p.id
AND pr.procedurerouteid = r.id
AND r.fromprocedurestepid = from_s.id
AND r.toprocedurestepid = to_s.id
)
TO '/Users/smethurstm/Documents/procedure-editor/reporting/ignored-routes.csv' DELIMITER ',' CSV HEADER;
A query for Jayne to check which routes are in more than one procedure. Includes only routes flagged as being included for export.
COPY (
SELECT
r.id AS route_id,
r.triplestoreid AS route_triplestore_id,
from_step.name AS from_step_name,
to_step.name AS to_step_name,
procedure_routes.procedure_count AS procedure_count,
STRING_AGG(procedure.name::text, ' | ') AS procedures
FROM procedure.procedureroute r
LEFT JOIN
(
SELECT pr.procedurerouteid, p.procedurename AS name
FROM procedure.procedurerouteprocedure pr, procedure.procedure p
WHERE pr.procedureid = p.id
AND pr.is_included_in_export is TRUE
) procedure
ON procedure.procedurerouteid = r.id
INNER JOIN (
SELECT s.id, s.procedurestepname AS name
FROM procedure.procedurestep s
) from_step
ON from_step.id = r.fromprocedurestepid
INNER JOIN (
SELECT s.id, s.procedurestepname AS name
FROM procedure.procedurestep s
) to_step
ON to_step.id = r.toprocedurestepid
INNER JOIN (
SELECT pr.procedurerouteid, count(pr.id) AS procedure_count
FROM procedure.procedurerouteprocedure pr
WHERE pr.is_included_in_export is TRUE
GROUP BY pr.procedurerouteid
) procedure_routes
ON procedure_routes.procedurerouteid = r.id
WHERE procedure_count > 1
GROUP BY r.id, from_step.name, to_step.name, procedure_count
ORDER BY procedure_count DESC
)
TO '/Users/smethurstm/Documents/procedure-editor/reporting/route-profileration.csv' DELIMITER ',' CSV HEADER;