[Notes] [Git][BuildStream/buildstream][danielsilverstone-ct/further-optimisations] _yaml.py: Remove use of isinstance() in `{node, list}_{chain_, }copy`

[Daniel Silverstone <gitlab mg gitlab com>]

Title: GitLab

Daniel Silverstone pushed to branch danielsilverstone-ct/further-optimisations at BuildStream / buildstream

Commits:

• a65e619e
  by Daniel Silverstone at 2018-11-09T13:21:00Z

  _yaml.py: Remove use of isinstance() in `{node,list}_{chain_,}copy`
  
  A non-trivial proportion of time pre-scheduler in `bst build` is spent
  copying (or chain-copying) nodes.  Approximately a quarter of the time
  spent in that effort is in `isinstance()`.  This removes that CPU load.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

1 changed file:

• buildstream/_yaml.py

Changes:

buildstream/_yaml.py

---

...	...	@@ -1066,16 +1066,48 @@ class ChainMap(collections.ChainMap):
1066	1066	except KeyError:
1067	1067	return default
1068	1068
	1069	+# Node copying
	1070	+#
	1071	+# Unfortunately we copy nodes a *lot* and `isinstance()` is super-slow when
	1072	+# things from collections.abc get involved. The result is the following
	1073	+# intricate but substantially faster group of tuples and the use of `in`.
	1074	+#
	1075	+# If any of the {node,list}_{chain_,}_copy routines raise a ValueError
	1076	+# then it's likely additional types need adding to these tuples.
	1077	+
	1078	+# When chaining a copy, these types are skipped since the ChainMap will
	1079	+# retrieve them from the source node when needed. Other copiers might copy
	1080	+# them, so we call them __quick_types.
	1081	+__quick_types = (str, bool,
	1082	+ yaml.scalarstring.PreservedScalarString,
	1083	+ yaml.scalarstring.SingleQuotedScalarString,
	1084	+ yaml.scalarstring.DoubleQuotedScalarString)
	1085	+
	1086	+# These types have to be iterated like a dictionary
	1087	+__dict_types = (dict, ChainMap, yaml.comments.CommentedMap)
	1088	+
	1089	+# These types have to be iterated like a list
	1090	+__list_types = (list, yaml.comments.CommentedSeq)
	1091	+
	1092	+# These are the provenance types, which have to be cloned rather than any other
	1093	+# copying tactic.
	1094	+__provenance_types = (Provenance, DictProvenance, MemberProvenance, ElementProvenance)
	1095	+
1069	1096
1070	1097	def node_chain_copy(source):
1071	1098	copy = ChainMap({}, source)
1072	1099	for key, value in source.items():
1073		- if isinstance(value, collections.Mapping):
	1100	+ value_type = type(value)
	1101	+ if value_type in __dict_types:
1074	1102	copy[key] = node_chain_copy(value)
1075		- elif isinstance(value, list):
	1103	+ elif value_type in __list_types:
1076	1104	copy[key] = list_chain_copy(value)
1077		- elif isinstance(value, Provenance):
	1105	+ elif value_type in __provenance_types:
1078	1106	copy[key] = value.clone()
	1107	+ elif value_type in __quick_types:
	1108	+ pass # No need to copy these, the chainmap deals with it
	1109	+ else:
	1110	+ raise ValueError("Unable to be quick about node_chain_copy of {}".format(value_type))
1079	1111
1080	1112	return copy
1081	1113
...	...	@@ -1083,14 +1115,17 @@ def node_chain_copy(source):
1083	1115	def list_chain_copy(source):
1084	1116	copy = []
1085	1117	for item in source:
1086		- if isinstance(item, collections.Mapping):
	1118	+ item_type = type(item)
	1119	+ if item_type in __dict_types:
1087	1120	copy.append(node_chain_copy(item))
1088		- elif isinstance(item, list):
	1121	+ elif item_type in __list_types:
1089	1122	copy.append(list_chain_copy(item))
1090		- elif isinstance(item, Provenance):
	1123	+ elif item_type in __provenance_types:
1091	1124	copy.append(item.clone())
1092		- else:
	1125	+ elif item_type in __quick_types:
1093	1126	copy.append(item)
	1127	+ else: # Fallback
	1128	+ raise ValueError("Unable to be quick about list_chain_copy of {}".format(item_type))
1094	1129
1095	1130	return copy
1096	1131
...	...	@@ -1098,14 +1133,17 @@ def list_chain_copy(source):
1098	1133	def node_copy(source):
1099	1134	copy = {}
1100	1135	for key, value in source.items():
1101		- if isinstance(value, collections.Mapping):
	1136	+ value_type = type(value)
	1137	+ if value_type in __dict_types:
1102	1138	copy[key] = node_copy(value)
1103		- elif isinstance(value, list):
	1139	+ elif value_type in __list_types:
1104	1140	copy[key] = list_copy(value)
1105		- elif isinstance(value, Provenance):
	1141	+ elif value_type in __provenance_types:
1106	1142	copy[key] = value.clone()
1107		- else:
	1143	+ elif value_type in __quick_types:
1108	1144	copy[key] = value
	1145	+ else:
	1146	+ raise ValueError("Unable to be quick about node_copy of {}".format(value_type))
1109	1147
1110	1148	ensure_provenance(copy)
1111	1149
...	...	@@ -1115,14 +1153,17 @@ def node_copy(source):
1115	1153	def list_copy(source):
1116	1154	copy = []
1117	1155	for item in source:
1118		- if isinstance(item, collections.Mapping):
	1156	+ item_type = type(item)
	1157	+ if item_type in __dict_types:
1119	1158	copy.append(node_copy(item))
1120		- elif isinstance(item, list):
	1159	+ elif item_type in __list_types:
1121	1160	copy.append(list_copy(item))
1122		- elif isinstance(item, Provenance):
	1161	+ elif item_type in __provenance_types:
1123	1162	copy.append(item.clone())
1124		- else:
	1163	+ elif item_type in __quick_types:
1125	1164	copy.append(item)
	1165	+ else:
	1166	+ raise ValueError("Unable to be quick about list_copy of {}".format(item_type))
1126	1167
1127	1168	return copy
1128	1169

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.