|
1
|
|
-# Copyright (C) 2018 Bloomberg LP
|
|
2
|
|
-#
|
|
3
|
|
-# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
|
-# you may not use this file except in compliance with the License.
|
|
5
|
|
-# You may obtain a copy of the License at
|
|
6
|
|
-#
|
|
7
|
|
-# <http://www.apache.org/licenses/LICENSE-2.0>
|
|
8
|
|
-#
|
|
9
|
|
-# Unless required by applicable law or agreed to in writing, software
|
|
10
|
|
-# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
|
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
|
-# See the License for the specific language governing permissions and
|
|
13
|
|
-# limitations under the License.
|
|
14
|
|
-
|
|
15
|
|
-
|
|
16
|
|
-"""
|
|
17
|
|
-ActionCache
|
|
18
|
|
-===========
|
|
19
|
|
-
|
|
20
|
|
-Implements a simple in-memory action cache.
|
|
21
|
|
-
|
|
22
|
|
-The action cache maps Action to their corresponding ActionResult. An
|
|
23
|
|
-ActionResult may be found in cache, for any given Action, if that action has
|
|
24
|
|
-already been executed.
|
|
25
|
|
-
|
|
26
|
|
-Note:
|
|
27
|
|
- Action and ActionResult are referenced by their Digest and mapping is stored
|
|
28
|
|
- in-memory.
|
|
29
|
|
-"""
|
|
30
|
|
-
|
|
31
|
|
-import collections
|
|
32
|
|
-
|
|
33
|
|
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
|
|
34
|
|
-
|
|
35
|
|
-
|
|
36
|
|
-class ActionCache:
|
|
37
|
|
- """In-memory Action to ActionResult associative array.
|
|
38
|
|
- """
|
|
39
|
|
-
|
|
40
|
|
- def __init__(self, storage, max_cached_actions):
|
|
41
|
|
- """Initialises a new ActionCache instance.
|
|
42
|
|
-
|
|
43
|
|
- Args:
|
|
44
|
|
- storage (StorageABC): storage backend instance to be used.
|
|
45
|
|
- max_cached_actions (int): maximun number of entries to cache.
|
|
46
|
|
- """
|
|
47
|
|
- self._storage = storage
|
|
48
|
|
- self._max_cached_actions = max_cached_actions
|
|
49
|
|
- self._digest_map = collections.OrderedDict()
|
|
50
|
|
-
|
|
51
|
|
- def get_action_result(self, action_digest):
|
|
52
|
|
- """Retrieves the cached ActionResult for the given Action digest.
|
|
53
|
|
-
|
|
54
|
|
- Args:
|
|
55
|
|
- action_digest (Digest): digest of the Action to query.
|
|
56
|
|
-
|
|
57
|
|
- Returns:
|
|
58
|
|
- The cached ActionResult matching the given Action digest or None if
|
|
59
|
|
- the nothing hass been cached yet for that Action.
|
|
60
|
|
- """
|
|
61
|
|
- key = (action_digest.hash, action_digest.size_bytes)
|
|
62
|
|
- if key in self._digest_map:
|
|
63
|
|
- action_result = self._storage.get_message(self._digest_map[key],
|
|
64
|
|
- re_pb2.ActionResult)
|
|
65
|
|
- if action_result is not None:
|
|
66
|
|
- if self._blobs_still_exist(action_result):
|
|
67
|
|
- self._digest_map.move_to_end(key)
|
|
68
|
|
- return action_result
|
|
69
|
|
- del self._digest_map[key]
|
|
70
|
|
- return None
|
|
71
|
|
-
|
|
72
|
|
- def put_action_result(self, action_digest, action_result):
|
|
73
|
|
- """Stores an ActionResult in cache for the given Action digest.
|
|
74
|
|
-
|
|
75
|
|
- If the cache size limit has been reached, the oldest cache entries will
|
|
76
|
|
- be dropped before insertion so that the cache size never exceeds the
|
|
77
|
|
- maximum numbers of entries allowed.
|
|
78
|
|
-
|
|
79
|
|
- Args:
|
|
80
|
|
- action_digest (Digest): digest of the Action to select.
|
|
81
|
|
- action_result (ActionResult): result object to store.
|
|
82
|
|
- """
|
|
83
|
|
- if self._max_cached_actions == 0:
|
|
84
|
|
- return
|
|
85
|
|
-
|
|
86
|
|
- while len(self._digest_map) >= self._max_cached_actions:
|
|
87
|
|
- self._digest_map.popitem(last=False)
|
|
88
|
|
-
|
|
89
|
|
- key = (action_digest.hash, action_digest.size_bytes)
|
|
90
|
|
- action_result_digest = self._storage.put_message(action_result)
|
|
91
|
|
- self._digest_map[key] = action_result_digest
|
|
92
|
|
-
|
|
93
|
|
- def _blobs_still_exist(self, action_result):
|
|
94
|
|
- """Checks CAS for ActionResult output blobs existance.
|
|
95
|
|
-
|
|
96
|
|
- Args:
|
|
97
|
|
- action_result (ActionResult): ActionResult to search referenced
|
|
98
|
|
- output blobs for.
|
|
99
|
|
-
|
|
100
|
|
- Returns:
|
|
101
|
|
- True if all referenced blobs are present in CAS, False otherwise.
|
|
102
|
|
- """
|
|
103
|
|
- blobs_needed = []
|
|
104
|
|
-
|
|
105
|
|
- for output_file in action_result.output_files:
|
|
106
|
|
- blobs_needed.append(output_file.digest)
|
|
107
|
|
-
|
|
108
|
|
- for output_directory in action_result.output_directories:
|
|
109
|
|
- blobs_needed.append(output_directory.tree_digest)
|
|
110
|
|
- tree = self._storage.get_message(output_directory.tree_digest,
|
|
111
|
|
- re_pb2.Tree)
|
|
112
|
|
- if tree is None:
|
|
113
|
|
- return False
|
|
114
|
|
- for file_node in tree.root.files:
|
|
115
|
|
- blobs_needed.append(file_node.digest)
|
|
116
|
|
- for child in tree.children:
|
|
117
|
|
- for file_node in child.files:
|
|
118
|
|
- blobs_needed.append(file_node.digest)
|
|
119
|
|
-
|
|
120
|
|
- if action_result.stdout_digest.hash and not action_result.stdout_raw:
|
|
121
|
|
- blobs_needed.append(action_result.stdout_digest)
|
|
122
|
|
- if action_result.stderr_digest.hash and not action_result.stderr_raw:
|
|
123
|
|
- blobs_needed.append(action_result.stderr_digest)
|
|
124
|
|
-
|
|
125
|
|
- missing = self._storage.missing_blobs(blobs_needed)
|
|
126
|
|
- return len(missing) == 0
|
