[Notes] [Git][BuildStream/buildstream][danielsilverstone-ct/further-optimisations] _yaml.py: Remove use of isinstance() in `{node, list}_{chain_, }copy`

[Daniel Silverstone <gitlab mg gitlab com>]

Title: GitLab

Daniel Silverstone pushed to branch danielsilverstone-ct/further-optimisations at BuildStream / buildstream

Commits:

• c5e9c904
  by Daniel Silverstone at 2018-11-09T11:28:48Z

  _yaml.py: Remove use of isinstance() in `{node,list}_{chain_,}copy`
  
  A non-trivial proportion of time pre-scheduler in `bst build` is spent
  copying (or chain-copying) nodes.  Approximately a quarter of the time
  spent in that effort is in `isinstance()`.  This removes that CPU load.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

1 changed file:

• buildstream/_yaml.py

Changes:

buildstream/_yaml.py

---

...	...	@@ -1066,16 +1066,47 @@ class ChainMap(collections.ChainMap):
1066	1066	except KeyError:
1067	1067	return default
1068	1068
	1069	+# Node copying
	1070	+#
	1071	+# Unfortunately we copy nodes a *lot* and `isinstance()` is super-slow when
	1072	+# things from collections.abc get involved. The result is the following
	1073	+# intricate but substantially faster group of tuples and the use of `in`.
	1074	+#
	1075	+# If any of the {node,list}_{chain_,}_copy routines raise a ValueError
	1076	+# then it's likely additional types need adding to these tuples.
	1077	+
	1078	+# When chaining a copy, these types are skipped since the ChainMap will
	1079	+# retrieve them from the source node when needed. Other copiers might copy
	1080	+# them, so we call them __quick_types.
	1081	+__quick_types = (str, bool,
	1082	+ yaml.scalarstring.PreservedScalarString,
	1083	+ yaml.scalarstring.SingleQuotedScalarString,
	1084	+ yaml.scalarstring.DoubleQuotedScalarString)
	1085	+
	1086	+# These types have to be iterated like a dictionary
	1087	+__dict_types = (dict, ChainMap, yaml.comments.CommentedMap)
	1088	+
	1089	+# These types have to be iterated like a list
	1090	+__list_types = (list, yaml.comments.CommentedSeq)
	1091	+
	1092	+# These are the provenance types, which have to be cloned rather than any other
	1093	+# copying tactic.
	1094	+__provenance_types = (Provenance, DictProvenance, MemberProvenance, ElementProvenance)
1069	1095
1070	1096	def node_chain_copy(source):
1071	1097	copy = ChainMap({}, source)
1072	1098	for key, value in source.items():
1073		- if isinstance(value, collections.Mapping):
	1099	+ value_type = type(value)
	1100	+ if value_type in __dict_types:
1074	1101	copy[key] = node_chain_copy(value)
1075		- elif isinstance(value, list):
	1102	+ elif value_type in __list_types:
1076	1103	copy[key] = list_chain_copy(value)
1077		- elif isinstance(value, Provenance):
	1104	+ elif value_type in __provenance_types:
1078	1105	copy[key] = value.clone()
	1106	+ elif value_type in __quick_types:
	1107	+ pass # No need to copy these, the chainmap deals with it
	1108	+ else:
	1109	+ raise ValueError("Unable to be quick about node_chain_copy of {}".format(value_type))
1079	1110
1080	1111	return copy
1081	1112
...	...	@@ -1083,14 +1114,17 @@ def node_chain_copy(source):
1083	1114	def list_chain_copy(source):
1084	1115	copy = []
1085	1116	for item in source:
1086		- if isinstance(item, collections.Mapping):
	1117	+ item_type = type(item)
	1118	+ if item_type in __dict_types:
1087	1119	copy.append(node_chain_copy(item))
1088		- elif isinstance(item, list):
	1120	+ elif item_type in __list_types:
1089	1121	copy.append(list_chain_copy(item))
1090		- elif isinstance(item, Provenance):
	1122	+ elif item_type in __provenance_types:
1091	1123	copy.append(item.clone())
1092		- else:
	1124	+ elif item_type in __quick_types:
1093	1125	copy.append(item)
	1126	+ else: # Fallback
	1127	+ raise ValueError("Unable to be quick about list_chain_copy of {}".format(item_type))
1094	1128
1095	1129	return copy
1096	1130
...	...	@@ -1098,14 +1132,17 @@ def list_chain_copy(source):
1098	1132	def node_copy(source):
1099	1133	copy = {}
1100	1134	for key, value in source.items():
1101		- if isinstance(value, collections.Mapping):
	1135	+ value_type = type(value)
	1136	+ if value_type in __dict_types:
1102	1137	copy[key] = node_copy(value)
1103		- elif isinstance(value, list):
	1138	+ elif value_type in __list_types:
1104	1139	copy[key] = list_copy(value)
1105		- elif isinstance(value, Provenance):
	1140	+ elif value_type in __provenance_types:
1106	1141	copy[key] = value.clone()
1107		- else:
	1142	+ elif value_type in __string_types:
1108	1143	copy[key] = value
	1144	+ else:
	1145	+ raise ValueError("Unable to be quick about node_copy of {}".format(value_type))
1109	1146
1110	1147	ensure_provenance(copy)
1111	1148
...	...	@@ -1115,14 +1152,17 @@ def node_copy(source):
1115	1152	def list_copy(source):
1116	1153	copy = []
1117	1154	for item in source:
1118		- if isinstance(item, collections.Mapping):
	1155	+ item_type = type(item)
	1156	+ if item_type in __dict_types:
1119	1157	copy.append(node_copy(item))
1120		- elif isinstance(item, list):
	1158	+ elif item_type in __list_types:
1121	1159	copy.append(list_copy(item))
1122		- elif isinstance(item, Provenance):
	1160	+ elif item_type in __provenance_types:
1123	1161	copy.append(item.clone())
1124		- else:
	1162	+ elif item_type in __string_types:
1125	1163	copy.append(item)
	1164	+ else:
	1165	+ raise ValueError("Unable to be quick about list_copy of {}".format(item_type))
1126	1166
1127	1167	return copy
1128	1168

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.