1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import time
23
24 from twisted.internet import reactor
25
26 from flumotion.common import log
27
28
29
30 MIN_REQUEST_SIZE = 64 * 1024 + 1
31
32 STATS_UPDATE_PERIOD = 10
33
34 CACHE_MISS = 0
35 CACHE_HIT = 1
36 TEMP_HIT = 2
37
38
40
42 self._stats = cacheStats
43 self._outdated = False
44 self._size = 0L
45 self._status = None
46 self.bytesReadFromSource = 0L
47 self.bytesReadFromCache = 0L
48
50 return self.bytesReadFromSource + self.bytesReadFromCache
51 bytesRead = property(getBytesRead)
52
54 total = self.bytesRead
55 if total == 0:
56 return 0.0
57 return float(self.bytesReadFromCache) / total
58 cacheReadRatio = property(getCacheReadRatio)
59
61 cs = self._stats
62 self._size = size
63 if cacheStatus == CACHE_HIT:
64 self._status = "cache-hit"
65 cs.cacheHitCount += 1
66 elif cacheStatus == TEMP_HIT:
67 cs.cacheHitCount += 1
68 cs.tempHitCount += 1
69 self._status = "temp-hit"
70 elif cacheStatus == CACHE_MISS:
71 cs.cacheMissCount += 1
72 if self._outdated:
73 self._status = "cache-outdate"
74 else:
75 self._status = "cache-miss"
76 cs._set("cache-hit-count", cs.cacheHitCount)
77 cs._set("temp-hit-count", cs.tempHitCount)
78 cs._set("cache-miss-count", cs.cacheMissCount)
79
81 self._outdated = True
82 cs = self._stats
83 cs.cacheOutdateCount += 1
84 cs._set("cache-outdate-count", cs.cacheOutdateCount)
85
86 - def onBytesRead(self, fromSource, fromCache, correction):
87 cs = self._stats
88 self.bytesReadFromSource += fromSource + correction
89 self.bytesReadFromCache += fromCache - correction
90 cs.bytesReadFromSource += fromSource + correction
91 cs.bytesReadFromCache += fromCache - correction
92
95
97 """
98 Provide the following log fields:
99 cache-status: value can be 'cache-miss', 'cache-outdate',
100 'cache-hit', or 'temp-hit'
101 cache-read: how many bytes where read from the cache for
102 this resource. the difference from resource-read
103 was read from the source file (network file system?)
104
105 The proportion read from cache and from source are adjusted
106 to take into account the file copy. It's done by remembering
107 how many bytes are copied at session level.
108 """
109 return {"cache-status": self._status,
110 "cache-read": self.bytesReadFromCache}
111
112
114
115 _updater = None
116 _callId = None
117
119
120 self._cacheUsage = 0
121 self._cacheUsageRatio = 0.0
122
123 self.cacheHitCount = 0
124 self.tempHitCount = 0
125 self.cacheMissCount = 0
126 self.cacheOutdateCount = 0
127 self.cleanupCount = 0
128
129 self.bytesReadFromSource = 0L
130 self.bytesReadFromCache = 0L
131
132 self.totalCopyCount = 0
133 self.currentCopyCount = 0
134 self.finishedCopyCount = 0
135 self.cancelledCopyCount = 0
136 self.bytesCopied = 0L
137 self._copyRatios = 0.0
138
140 self._updater = updater
141 if updater and (self._callId is None):
142 self._set("cache-usage-estimation", self._cacheUsage)
143 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
144 self._set("cleanup-count", self.cleanupCount)
145 self._set("last-cleanup-time", time.time())
146 self._set("current-copy-count", self.currentCopyCount)
147 self._set("finished-copy-count", self.finishedCopyCount)
148 self._set("cancelled-copy-count", self.cancelledCopyCount)
149 self._set("mean-copy-ratio", self.meanCopyRatio)
150 self._set("mean-bytes-copied", self.meanBytesCopied)
151 self._update()
152
158
160 total = self.bytesReadFromSource + self.bytesReadFromCache
161 if total == 0:
162 return 0
163 return float(self.bytesReadFromCache) / total
164 cacheReadRatio = property(getCacheReadRatio)
165
167 if self.finishedCopyCount == 0:
168 return 0
169 return self.bytesCopied / self.finishedCopyCount
170 meanBytesCopied = property(getMeanBytesCopied)
171
173 if self.finishedCopyCount == 0:
174 return 0
175 return self._copyRatios / self.finishedCopyCount
176 meanCopyRatio = property(getMeanCopyRatio)
177
179 self._cacheUsage = usage
180 self._cacheUsageRatio = float(usage) / max
181 self._set("cache-usage-estimation", self._cacheUsage)
182 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
183
185 self.cleanupCount += 1
186 self._set("cleanup-count", self.cleanupCount)
187 self._set("last-cleanup-time", time.time())
188
190 self.currentCopyCount += 1
191 self.totalCopyCount += 1
192 self._set("current-copy-count", self.currentCopyCount)
193
195 self.currentCopyCount -= 1
196 self.finishedCopyCount += 1
197 self.cancelledCopyCount += 1
198 self.bytesCopied += copied
199 self._copyRatios += float(copied) / size
200 self._set("current-copy-count", self.currentCopyCount)
201 self._set("finished-copy-count", self.finishedCopyCount)
202 self._set("cancelled-copy-count", self.cancelledCopyCount)
203 self._set("mean-copy-ratio", self.meanCopyRatio)
204 self._set("mean-bytes-copied", self.meanBytesCopied)
205
207 self.currentCopyCount -= 1
208 self.finishedCopyCount += 1
209 self.bytesCopied += size
210 self._copyRatios += 1.0
211 self._set("current-copy-count", self.currentCopyCount)
212 self._set("finished-copy-count", self.finishedCopyCount)
213 self._set("mean-copy-ratio", self.meanCopyRatio)
214 self._set("mean-bytes-copied", self.meanBytesCopied)
215
216 - def _set(self, key, value):
219
224
226 """
227 Statistic fields names:
228 CRR: Cache Read Ratio
229 CMC: Cache Miss Count
230 CHC: Cache Hit Count
231 THC: Temp Hit Count
232 COC: Cache Outdate Count
233 CCC: Cache Cleanup Count
234 CCU: Cache Current Usage
235 CUR: Cache Usage Ratio
236 PTC: coPy Total Count
237 PCC: coPy Current Count
238 PAC: coPy cAncellation Count
239 MCS: Mean Copy Size
240 MCR: Mean Copy Ratio
241 """
242 log.debug("stats-local-cache",
243 "CRR: %.4f; CMC: %d; CHC: %d; THC: %d; COC: %d; "
244 "CCC: %d; CCU: %d; CUR: %.5f; "
245 "PTC: %d; PCC: %d; PAC: %d; MCS: %d; MCR: %.4f",
246 self.cacheReadRatio, self.cacheMissCount,
247 self.cacheHitCount, self.tempHitCount,
248 self.cacheOutdateCount, self.cleanupCount,
249 self._cacheUsage, self._cacheUsageRatio,
250 self.totalCopyCount, self.currentCopyCount,
251 self.cancelledCopyCount, self.meanBytesCopied,
252 self.meanCopyRatio)
253