Paste #80

Paste Details

reply | raw

posted on May 14, 2010 7:31:30 AM

diff -uNr google_appengine_1.3.3/README google_appengine_1.3.4_prerelease/README
google_appengine_1.3.3/README [2010-04-20 03:02:10.000000000 +0900]
google_appengine_1.3.4_prerelease/README [2010-05-13 06:32:44.000000000 +0900]
54 54 Address to which this server should bind. (Default
55 55 localhost).
56 56 --port=PORT, -p PORT Port for the server to run on. (Default 8080)
57 --blobstore_path=PATH Path to use for storing Blobstore file stub data.
57 58 --datastore_path=PATH Path to use for storing Datastore file stub data.
58 59 (Default /tmp/dev_appserver.datastore)
60 --use_sqlite Use the new, SQLite based datastore stub.
61 (Default false)
59 62 --history_path=PATH Path to use for storing Datastore history.
60 63 (Default /tmp/dev_appserver.datastore.history)
61 64 --require_indexes Disallows queries that require composite indexes
…
79 82 --debug_imports Enables debug logging for module imports, showing
80 83 search paths used for finding modules and any
81 84 errors encountered during the import process.
85 --allow_skipped_files Allow access to files matched by app.yaml's
86 skipped_files (default False)
82 87 --disable_static_caching Never allow the browser to cache static files.
83 88 (Default enable if expiration set in app.yaml)
89 --disable_task_running When supplied, tasks will not be automatically
90 run after submission and must be run manually
91 in the local admin console.
92 --task_retry_seconds How long to wait in seconds before retrying a
93 task after it fails during execution.
94 (Default '30')
84 95
85 96
86 97 
diff -uNr google_appengine_1.3.3/RELEASE_NOTES google_appengine_1.3.4_prerelease/RELEASE_NOTES
google_appengine_1.3.3/RELEASE_NOTES [2010-04-20 03:02:10.000000000 +0900]
google_appengine_1.3.4_prerelease/RELEASE_NOTES [2010-05-13 06:32:43.000000000 +0900]
3 3
4 4 App Engine Python SDK - Release Notes
5 5
6 Version 1.3.4
7 =================================
8 - New bulkloader configuration syntax and wizard for easier import/export with
9 the datastore.
10 - Applications can now be configured to authenticate with OpenID by selecting
11 the OpenID option when creating your application in the admin console.
12 http://code.google.com/p/googleappengine/issues/detail?id=248
13 http://code.google.com/p/googleappengine/issues/detail?id=56
14 - New API to allow App Engine apps to act as OAuth service providers.
15 http://code.google.com/p/googleappengine/issues/detail?id=919
16 - Auto task execution is now enabled in the dev_appserver. To turn this off
17 use the flag --disable_task_running.
18 - Fixed an issue using db.put() with constructor initialized id based keys.
19 http://code.google.com/p/googleappengine/issues/detail?id=3209
20
6 21 Version 1.3.3
7 22 =================================
8 23 - A new experimental feature allows you to set dev_appserver datastore file 
diff -uNr google_appengine_1.3.3/VERSION google_appengine_1.3.4_prerelease/VERSION
google_appengine_1.3.3/VERSION [2010-04-20 03:02:10.000000000 +0900]
google_appengine_1.3.4_prerelease/VERSION [2010-05-13 07:29:11.000000000 +0900]
1 release: "1.3.3"
2 timestamp: 1270494723
1 release: "prerelease-1.3.4"
2 timestamp: 1272392128
3 3 api_versions: ['1'] 
diff -uNr google_appengine_1.3.3/google/appengine/api/datastore_types.py google_appengine_1.3.4_prerelease/google/appengine/api/datastore_types.py
google_appengine_1.3.3/google/appengine/api/datastore_types.py [2010-04-20 03:02:32.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/datastore_types.py [2010-05-13 06:32:53.000000000 +0900]
118 118 negative_ok=False):
119 119 """Raises an exception if value is not a valid integer.
120 120
121 An integer is valid if it's not negative or empty and is an integer.
122 The exception type can be specified with the exception argument;
123 it defaults to BadValueError.
121 An integer is valid if it's not negative or empty and is an integer
122 (either int or long). The exception type raised can be specified
123 with the exception argument; it defaults to BadValueError.
124 124
125 125 Args:
126 126 value: the value to validate.
…
132 132 """
133 133 if value is None and empty_ok:
134 134 return
135 if not isinstance(value, int):
135 if not isinstance(value, (int, long)):
136 136 raise exception('%s should be an integer; received %s (a %s).' %
137 137 (name, value, typename(value)))
138 138 if not value and not zero_ok:
…
1374 1374 pbvalue.mutable_uservalue().set_obfuscated_gaiaid(
1375 1375 value.user_id().encode('utf-8'))
1376 1376
1377 if value.federated_identity() is not None:
1378 pbvalue.mutable_uservalue().set_federated_identity(
1379 value.federated_identity().encode('utf-8'))
1380
1381 if value.federated_provider() is not None:
1382 pbvalue.mutable_uservalue().set_federated_provider(
1383 value.federated_provider().encode('utf-8'))
1384
1377 1385
1378 1386 def PackKey(name, value, pbvalue):
1379 1387 """Packs a reference property into a entity_pb.PropertyValue.
…
1577 1585 auth_domain = unicode(pbval.uservalue().auth_domain().decode('utf-8'))
1578 1586 obfuscated_gaiaid = pbval.uservalue().obfuscated_gaiaid().decode('utf-8')
1579 1587 obfuscated_gaiaid = unicode(obfuscated_gaiaid)
1588
1589 federated_identity = None
1590 if pbval.uservalue().has_federated_identity():
1591 federated_identity = unicode(
1592 pbval.uservalue().federated_identity().decode('utf-8'))
1593
1580 1594 value = users.User(email=email,
1581 1595 _auth_domain=auth_domain,
1582 _user_id=obfuscated_gaiaid)
1596 _user_id=obfuscated_gaiaid,
1597 federated_identity=federated_identity)
1583 1598 else:
1584 1599 value = None
1585 1600
…
1703 1718 elif type_ == type(None):
1704 1719 return None
1705 1720 return type_(value_string)
1706 
diff -uNr google_appengine_1.3.3/google/appengine/api/labs/taskqueue/taskqueue_stub.py google_appengine_1.3.4_prerelease/google/appengine/api/labs/taskqueue/taskqueue_stub.py
google_appengine_1.3.3/google/appengine/api/labs/taskqueue/taskqueue_stub.py [2010-04-20 03:02:33.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/labs/taskqueue/taskqueue_stub.py [2010-05-13 06:32:56.000000000 +0900]
17 17
18 18 """Stub version of the Task Queue API.
19 19
20 This stub only stores tasks; it doesn't actually run them. It also validates
21 the tasks by checking their queue name against the queue.yaml.
20 This stub stores tasks and runs them via dev_appserver's AddEvent capability.
21 It also validates the tasks by checking their queue name against the queue.yaml.
22 22
23 23 As well as implementing Task Queue API functions, the stub exposes various other
24 24 functions that are used by the dev_appserver's admin console to display the
…
28 28
29 29
30 30
31 import StringIO
31 32 import base64
32 33 import bisect
33 34 import datetime
…
300 301 class TaskQueueServiceStub(apiproxy_stub.APIProxyStub):
301 302 """Python only task queue service stub.
302 303
303 This stub does not attempt to automatically execute tasks. Instead, it
304 stores them for display on a console. The user may manually execute the
305 tasks from the console.
304 This stub executes tasks when enabled by using the dev_appserver's AddEvent
305 capability. When task running is disabled this stub will store tasks for
306 display on a console, where the user may manually execute the tasks.
306 307 """
307 308
308 309 queue_yaml_parser = _ParseQueueYaml
309 310
310 def __init__(self, service_name='taskqueue', root_path=None):
311 def __init__(self,
312 service_name='taskqueue',
313 root_path=None,
314 auto_task_running=False,
315 task_retry_seconds=30):
311 316 """Constructor.
312 317
313 318 Args:
…
315 320 root_path: Root path to the directory of the application which may contain
316 321 a queue.yaml file. If None, then it's assumed no queue.yaml file is
317 322 available.
323 auto_task_running: When True, the dev_appserver should automatically
324 run tasks after they are enqueued.
325 task_retry_seconds: How long to wait between task executions after a
326 task fails.
318 327 """
319 328 super(TaskQueueServiceStub, self).__init__(service_name)
320 329 self._taskqueues = {}
321 330 self._next_task_id = 1
322 331 self._root_path = root_path
323 332
333 self._add_event = None
334 self._auto_task_running = auto_task_running
335 self._task_retry_seconds = task_retry_seconds
336
324 337 self._app_queues = {}
325 338
326 339 def _ChooseTaskName(self):
…
480 493 taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS)
481 494 else:
482 495 existing_tasks.append(add_request)
496
497 if self._add_event and self._auto_task_running:
498 self._add_event(
499 add_request.eta_usec() / 1000000.0,
500 lambda: self._RunTask(
501 add_request.queue_name(), add_request.task_name()))
502
483 503 existing_tasks.sort(_CompareTasksByEta)
484 504
485 505 def _IsValidQueue(self, queue_name):
…
503 523 return True
504 524 return False
505 525
526 def _RunTask(self, queue_name, task_name):
527 """Returns a fake request for running a task in the dev_appserver.
528
529 Args:
530 queue_name: The queue the task is in.
531 task_name: The name of the task to run.
532
533 Returns:
534 None if this task no longer exists or tuple (connection, addrinfo) of
535 a fake connection and address information used to run this task. The
536 task will be deleted after it runs or re-enqueued in the future on
537 failure.
538 """
539 task_list = self.GetTasks(queue_name)
540 for task in task_list:
541 if task['name'] == task_name:
542 break
543 else:
544 return None
545
546 class FakeConnection(object):
547 def __init__(self, input_buffer):
548 self.rfile = StringIO.StringIO(input_buffer)
549 self.wfile = StringIO.StringIO()
550 self.wfile_close = self.wfile.close
551 self.wfile.close = self.connection_done
552
553 def connection_done(myself):
554 result = myself.wfile.getvalue()
555 myself.wfile_close()
556 first_line, rest = result.split('\n', 1)
557 version, code, rest = first_line.split(' ', 2)
558 if 200 <= int(code) <= 299:
559 self.DeleteTask(queue_name, task_name)
560 return
561
562 logging.warning('Task named "%s" on queue "%s" failed with code %s; '
563 'will retry in %d seconds',
564 task_name, queue_name, code, self._task_retry_seconds)
565 self._add_event(
566 time.time() + self._task_retry_seconds,
567 lambda: self._RunTask(queue_name, task_name))
568
569 def close(self):
570 pass
571
572 def makefile(self, mode, buffsize):
573 if mode.startswith('w'):
574 return self.wfile
575 else:
576 return self.rfile
577
578 payload = StringIO.StringIO()
579 payload.write('%s %s HTTP/1.1\r\n' % (task['method'], task['url']))
580 for key, value in task['headers']:
581 payload.write('%s: %s\r\n' % (key, value))
582 payload.write('\r\n')
583 payload.write(task['body'])
584
585 return FakeConnection(payload.getvalue()), ('0.1.0.2', 80)
586
506 587 def GetQueues(self):
507 588 """Gets all the applications's queues.
508 589 
diff -uNr google_appengine_1.3.3/google/appengine/api/oauth/__init__.py google_appengine_1.3.4_prerelease/google/appengine/api/oauth/__init__.py
google_appengine_1.3.3/google/appengine/api/oauth/__init__.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/oauth/__init__.py [2010-05-13 06:32:55.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """OAuth API module."""
19
20 from oauth_api import *
21
22
23 __all__ = ['Error',
24 'OAuthRequestError',
25 'NotAllowedError',
26 'InvalidOAuthParametersError',
27 'InvalidOAuthTokenError',
28 'OAuthServiceFailureError',
29 'get_current_user',
30 'is_current_user_admin',
31 'get_oauth_consumer_key',
32 ] 
diff -uNr google_appengine_1.3.3/google/appengine/api/oauth/oauth_api.py google_appengine_1.3.4_prerelease/google/appengine/api/oauth/oauth_api.py
google_appengine_1.3.3/google/appengine/api/oauth/oauth_api.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/oauth/oauth_api.py [2010-05-13 06:32:55.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """OAuth API.
19
20 A service that enables App Engine apps to validate OAuth requests.
21
22 Classes defined here:
23 Error: base exception type
24 NotAllowedError: OAuthService exception
25 OAuthRequestError: OAuthService exception
26 InvalidOAuthParametersError: OAuthService exception
27 InvalidOAuthTokenError: OAuthService exception
28 OAuthServiceFailureError: OAuthService exception
29 """
30
31
32
33
34
35
36
37 import os
38
39 from google.appengine.api import apiproxy_stub_map
40 from google.appengine.api import user_service_pb
41 from google.appengine.api import users
42 from google.appengine.runtime import apiproxy_errors
43
44
45 class Error(Exception):
46 """Base error class for this module."""
47
48
49 class OAuthRequestError(Error):
50 """Base error type for invalid OAuth requests."""
51
52
53 class NotAllowedError(OAuthRequestError):
54 """Raised if the requested URL does not permit OAuth authentication."""
55
56
57 class InvalidOAuthParametersError(OAuthRequestError):
58 """Raised if the request was a malformed OAuth request.
59
60 For example, the request may have omitted a required parameter, contained
61 an invalid signature, or was made by an unknown consumer.
62 """
63
64
65 class InvalidOAuthTokenError(OAuthRequestError):
66 """Raised if the request contained an invalid token.
67
68 For example, the token may have been revoked by the user.
69 """
70
71
72 class OAuthServiceFailureError(Error):
73 """Raised if there was a problem communicating with the OAuth service."""
74
75
76 def get_current_user():
77 """Returns the User on whose behalf the request was made.
78
79 Returns:
80 User
81
82 Raises:
83 OAuthRequestError: The request was not a valid OAuth request.
84 OAuthServiceFailureError: An unknown error occurred.
85 """
86 _maybe_call_get_oauth_user()
87 return _get_user_from_environ()
88
89
90 def is_current_user_admin():
91 """Returns true if the User on whose behalf the request was made is an admin.
92
93 Returns:
94 boolean
95
96 Raises:
97 OAuthRequestError: The request was not a valid OAuth request.
98 OAuthServiceFailureError: An unknown error occurred.
99 """
100 _maybe_call_get_oauth_user()
101 return os.environ.get('OAUTH_IS_ADMIN', '0') == '1'
102
103
104 def get_oauth_consumer_key():
105 """Returns the value of the 'oauth_consumer_key' parameter from the request.
106
107 Returns:
108 string: The value of the 'oauth_consumer_key' parameter from the request,
109 an identifier for the consumer that signed the request.
110
111 Raises:
112 OAuthRequestError: The request was not a valid OAuth request.
113 OAuthServiceFailureError: An unknown error occurred.
114 """
115 req = user_service_pb.CheckOAuthSignatureRequest()
116 resp = user_service_pb.CheckOAuthSignatureResponse()
117 try:
118 apiproxy_stub_map.MakeSyncCall('user', 'CheckOAuthSignature', req, resp)
119 except apiproxy_errors.ApplicationError, e:
120 if (e.application_error ==
121 user_service_pb.UserServiceError.OAUTH_INVALID_REQUEST):
122 raise InvalidOAuthParametersError
123 elif (e.application_error ==
124 user_service_pb.UserServiceError.OAUTH_ERROR):
125 raise OAuthServiceFailureError
126 else:
127 raise OAuthServiceFailureError
128 return resp.oauth_consumer_key()
129
130
131 def _maybe_call_get_oauth_user():
132 """Makes an GetOAuthUser RPC and stores the results in os.environ.
133
134 This method will only make the RPC if 'OAUTH_ERROR_CODE' has not already
135 been set.
136 """
137 if 'OAUTH_ERROR_CODE' not in os.environ:
138 req = user_service_pb.GetOAuthUserRequest()
139 resp = user_service_pb.GetOAuthUserResponse()
140 try:
141 apiproxy_stub_map.MakeSyncCall('user', 'GetOAuthUser', req, resp)
142 os.environ['OAUTH_EMAIL'] = resp.email()
143 os.environ['OAUTH_AUTH_DOMAIN'] = resp.auth_domain()
144 os.environ['OAUTH_USER_ID'] = resp.user_id()
145 if resp.is_admin():
146 os.environ['OAUTH_IS_ADMIN'] = '1'
147 else:
148 os.environ['OAUTH_IS_ADMIN'] = '0'
149 os.environ['OAUTH_ERROR_CODE'] = ''
150 except apiproxy_errors.ApplicationError, e:
151 os.environ['OAUTH_ERROR_CODE'] = str(e.application_error)
152 _maybe_raise_exception()
153
154
155 def _maybe_raise_exception():
156 """Raises an error if one has been stored in os.environ.
157
158 This method requires that 'OAUTH_ERROR_CODE' has already been set (an empty
159 string indicates that there is no actual error).
160 """
161 assert 'OAUTH_ERROR_CODE' in os.environ
162 error = os.environ['OAUTH_ERROR_CODE']
163 if error:
164 if error == str(user_service_pb.UserServiceError.NOT_ALLOWED):
165 raise NotAllowedError
166 elif error == str(user_service_pb.UserServiceError.OAUTH_INVALID_REQUEST):
167 raise InvalidOAuthParametersError
168 elif error == str(user_service_pb.UserServiceError.OAUTH_INVALID_TOKEN):
169 raise InvalidOAuthTokenError
170 elif error == str(user_service_pb.UserServiceError.OAUTH_ERROR):
171 raise OAuthServiceFailureError
172 else:
173 raise OAuthServiceFailureError
174
175
176 def _get_user_from_environ():
177 """Returns a User based on values stored in os.environ.
178
179 This method requires that 'OAUTH_EMAIL', 'OAUTH_AUTH_DOMAIN', and
180 'OAUTH_USER_ID' have already been set.
181
182 Returns:
183 User
184 """
185 assert 'OAUTH_EMAIL' in os.environ
186 assert 'OAUTH_AUTH_DOMAIN' in os.environ
187 assert 'OAUTH_USER_ID' in os.environ
188 return users.User(email=os.environ['OAUTH_EMAIL'],
189 _auth_domain=os.environ['OAUTH_AUTH_DOMAIN'],
190 _user_id=os.environ['OAUTH_USER_ID']) 
diff -uNr google_appengine_1.3.3/google/appengine/api/user_service_pb.py google_appengine_1.3.4_prerelease/google/appengine/api/user_service_pb.py
google_appengine_1.3.3/google/appengine/api/user_service_pb.py [2010-04-20 03:02:16.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/user_service_pb.py [2010-05-13 06:32:35.000000000 +0900]
102 102 destination_url_ = ""
103 103 has_auth_domain_ = 0
104 104 auth_domain_ = ""
105 has_federated_identity_ = 0
106 federated_identity_ = "google.com"
105 107
106 108 def __init__(self, contents=None):
107 109 if contents is not None: self.MergeFromString(contents)
…
132 134
133 135 def has_auth_domain(self): return self.has_auth_domain_
134 136
137 def federated_identity(self): return self.federated_identity_
138
139 def set_federated_identity(self, x):
140 self.has_federated_identity_ = 1
141 self.federated_identity_ = x
142
143 def clear_federated_identity(self):
144 if self.has_federated_identity_:
145 self.has_federated_identity_ = 0
146 self.federated_identity_ = "google.com"
147
148 def has_federated_identity(self): return self.has_federated_identity_
149
135 150
136 151 def MergeFrom(self, x):
137 152 assert x is not self
138 153 if (x.has_destination_url()): self.set_destination_url(x.destination_url())
139 154 if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain())
155 if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
140 156
141 157 def Equals(self, x):
142 158 if x is self: return 1
…
144 160 if self.has_destination_url_ and self.destination_url_ != x.destination_url_: return 0
145 161 if self.has_auth_domain_ != x.has_auth_domain_: return 0
146 162 if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0
163 if self.has_federated_identity_ != x.has_federated_identity_: return 0
164 if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
147 165 return 1
148 166
149 167 def IsInitialized(self, debug_strs=None):
…
158 176 n = 0
159 177 n += self.lengthString(len(self.destination_url_))
160 178 if (self.has_auth_domain_): n += 1 + self.lengthString(len(self.auth_domain_))
179 if (self.has_federated_identity_): n += 1 + self.lengthString(len(self.federated_identity_))
161 180 return n + 1
162 181
163 182 def Clear(self):
164 183 self.clear_destination_url()
165 184 self.clear_auth_domain()
185 self.clear_federated_identity()
166 186
167 187 def OutputUnchecked(self, out):
168 188 out.putVarInt32(10)
…
170 190 if (self.has_auth_domain_):
171 191 out.putVarInt32(18)
172 192 out.putPrefixedString(self.auth_domain_)
193 if (self.has_federated_identity_):
194 out.putVarInt32(26)
195 out.putPrefixedString(self.federated_identity_)
173 196
174 197 def TryMerge(self, d):
175 198 while d.avail() > 0:
…
180 203 if tt == 18:
181 204 self.set_auth_domain(d.getPrefixedString())
182 205 continue
206 if tt == 26:
207 self.set_federated_identity(d.getPrefixedString())
208 continue
183 209 if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
184 210 d.skipData(tt)
185 211
…
188 214 res=""
189 215 if self.has_destination_url_: res+=prefix+("destination_url: %s\n" % self.DebugFormatString(self.destination_url_))
190 216 if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_))
217 if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
191 218 return res
192 219
193 220
…
196 223
197 224 kdestination_url = 1
198 225 kauth_domain = 2
226 kfederated_identity = 3
199 227
200 228 _TEXT = _BuildTagLookupTable({
201 229 0: "ErrorCode",
202 230 1: "destination_url",
203 231 2: "auth_domain",
204 }, 2)
232 3: "federated_identity",
233 }, 3)
205 234
206 235 _TYPES = _BuildTagLookupTable({
207 236 0: ProtocolBuffer.Encoder.NUMERIC,
208 237 1: ProtocolBuffer.Encoder.STRING,
209 238 2: ProtocolBuffer.Encoder.STRING,
210 }, 2, ProtocolBuffer.Encoder.MAX_TYPE)
239 3: ProtocolBuffer.Encoder.STRING,
240 }, 3, ProtocolBuffer.Encoder.MAX_TYPE)
211 241
212 242 _STYLE = """"""
213 243 _STYLE_CONTENT_TYPE = """"""
…
556 586 auth_domain_ = ""
557 587 has_user_organization_ = 0
558 588 user_organization_ = ""
589 has_is_admin_ = 0
590 is_admin_ = 0
559 591
560 592 def __init__(self, contents=None):
561 593 if contents is not None: self.MergeFromString(contents)
…
612 644
613 645 def has_user_organization(self): return self.has_user_organization_
614 646
647 def is_admin(self): return self.is_admin_
648
649 def set_is_admin(self, x):
650 self.has_is_admin_ = 1
651 self.is_admin_ = x
652
653 def clear_is_admin(self):
654 if self.has_is_admin_:
655 self.has_is_admin_ = 0
656 self.is_admin_ = 0
657
658 def has_is_admin(self): return self.has_is_admin_
659
615 660
616 661 def MergeFrom(self, x):
617 662 assert x is not self
…
619 664 if (x.has_user_id()): self.set_user_id(x.user_id())
620 665 if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain())
621 666 if (x.has_user_organization()): self.set_user_organization(x.user_organization())
667 if (x.has_is_admin()): self.set_is_admin(x.is_admin())
622 668
623 669 def Equals(self, x):
624 670 if x is self: return 1
…
630 676 if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0
631 677 if self.has_user_organization_ != x.has_user_organization_: return 0
632 678 if self.has_user_organization_ and self.user_organization_ != x.user_organization_: return 0
679 if self.has_is_admin_ != x.has_is_admin_: return 0
680 if self.has_is_admin_ and self.is_admin_ != x.is_admin_: return 0
633 681 return 1
634 682
635 683 def IsInitialized(self, debug_strs=None):
…
654 702 n += self.lengthString(len(self.user_id_))
655 703 n += self.lengthString(len(self.auth_domain_))
656 704 if (self.has_user_organization_): n += 1 + self.lengthString(len(self.user_organization_))
705 if (self.has_is_admin_): n += 2
657 706 return n + 3
658 707
659 708 def Clear(self):
…
661 710 self.clear_user_id()
662 711 self.clear_auth_domain()
663 712 self.clear_user_organization()
713 self.clear_is_admin()
664 714
665 715 def OutputUnchecked(self, out):
666 716 out.putVarInt32(10)
…
672 722 if (self.has_user_organization_):
673 723 out.putVarInt32(34)
674 724 out.putPrefixedString(self.user_organization_)
725 if (self.has_is_admin_):
726 out.putVarInt32(40)
727 out.putBoolean(self.is_admin_)
675 728
676 729 def TryMerge(self, d):
677 730 while d.avail() > 0:
…
688 741 if tt == 34:
689 742 self.set_user_organization(d.getPrefixedString())
690 743 continue
744 if tt == 40:
745 self.set_is_admin(d.getBoolean())
746 continue
691 747 if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
692 748 d.skipData(tt)
693 749
…
698 754 if self.has_user_id_: res+=prefix+("user_id: %s\n" % self.DebugFormatString(self.user_id_))
699 755 if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_))
700 756 if self.has_user_organization_: res+=prefix+("user_organization: %s\n" % self.DebugFormatString(self.user_organization_))
757 if self.has_is_admin_: res+=prefix+("is_admin: %s\n" % self.DebugFormatBool(self.is_admin_))
701 758 return res
702 759
703 760
…
708 765 kuser_id = 2
709 766 kauth_domain = 3
710 767 kuser_organization = 4
768 kis_admin = 5
711 769
712 770 _TEXT = _BuildTagLookupTable({
713 771 0: "ErrorCode",
…
715 773 2: "user_id",
716 774 3: "auth_domain",
717 775 4: "user_organization",
718 }, 4)
776 5: "is_admin",
777 }, 5)
719 778
720 779 _TYPES = _BuildTagLookupTable({
721 780 0: ProtocolBuffer.Encoder.NUMERIC,
…
723 782 2: ProtocolBuffer.Encoder.STRING,
724 783 3: ProtocolBuffer.Encoder.STRING,
725 784 4: ProtocolBuffer.Encoder.STRING,
726 }, 4, ProtocolBuffer.Encoder.MAX_TYPE)
785 5: ProtocolBuffer.Encoder.NUMERIC,
786 }, 5, ProtocolBuffer.Encoder.MAX_TYPE)
727 787
728 788 _STYLE = """"""
729 789 _STYLE_CONTENT_TYPE = """""" 
diff -uNr google_appengine_1.3.3/google/appengine/api/user_service_stub.py google_appengine_1.3.4_prerelease/google/appengine/api/user_service_stub.py
google_appengine_1.3.3/google/appengine/api/user_service_stub.py [2010-04-20 03:02:36.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/user_service_stub.py [2010-05-13 06:32:55.000000000 +0900]
28 28 _DEFAULT_LOGIN_URL = 'https://www.google.com/accounts/Login?continue=%s'
29 29 _DEFAULT_LOGOUT_URL = 'https://www.google.com/accounts/Logout?continue=%s'
30 30
31 _OAUTH_CONSUMER_KEY = 'example.com'
32 _OAUTH_EMAIL = 'example@example.com'
33 _OAUTH_USER_ID = '0'
34 _OAUTH_AUTH_DOMAIN = 'gmail.com'
35
31 36
32 37 class UserServiceStub(apiproxy_stub.APIProxyStub):
33 38 """Trivial implementation of the UserService."""
…
61 66 """Trivial implementation of UserService.CreateLoginURL().
62 67
63 68 Args:
64 request: the URL to redirect to after login; a base.StringProto
65 response: the login URL; a base.StringProto
69 request: a CreateLoginURLRequest
70 response: a CreateLoginURLResponse
66 71 """
67 72 self.__num_requests += 1
68 73 response.set_login_url(
…
73 78 """Trivial implementation of UserService.CreateLogoutURL().
74 79
75 80 Args:
76 request: the URL to redirect to after logout; a base.StringProto
77 response: the logout URL; a base.StringProto
81 request: a CreateLogoutURLRequest
82 response: a CreateLogoutURLResponse
78 83 """
79 84 self.__num_requests += 1
80 85 response.set_logout_url(
81 86 self._logout_url %
82 87 urllib.quote(self._AddHostToContinueURL(request.destination_url())))
83 88
89 def _Dynamic_GetOAuthUser(self, unused_request, response):
90 """Trivial implementation of UserService.GetOAuthUser().
91
92 Args:
93 unused_request: a GetOAuthUserRequest
94 response: a GetOAuthUserResponse
95 """
96 self.__num_requests += 1
97 response.set_email(_OAUTH_EMAIL)
98 response.set_user_id(_OAUTH_USER_ID)
99 response.set_auth_domain(_OAUTH_AUTH_DOMAIN)
100
101 def _Dynamic_CheckOAuthSignature(self, unused_request, response):
102 """Trivial implementation of UserService.CheckOAuthSignature().
103
104 Args:
105 unused_request: a CheckOAuthSignatureRequest
106 response: a CheckOAuthSignatureResponse
107 """
108 self.__num_requests += 1
109 response.set_oauth_consumer_key(_OAUTH_CONSUMER_KEY)
110
84 111 def _AddHostToContinueURL(self, continue_url):
85 112 """Adds the request host to the continue url if no host is specified.
86 113 
diff -uNr google_appengine_1.3.3/google/appengine/api/users.py google_appengine_1.3.4_prerelease/google/appengine/api/users.py
google_appengine_1.3.3/google/appengine/api/users.py [2010-04-20 03:02:36.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/api/users.py [2010-05-13 06:32:53.000000000 +0900]
18 18 """Python datastore class User to be used as a datastore data type.
19 19
20 20 Classes defined here:
21 User: object representing a user.
21 User: object representing a user. A user could be a Google Accounts user
22 or a federated user.
22 23 Error: base exception type
23 24 UserNotFoundError: UserService exception
24 25 RedirectTooLongError: UserService exception
…
29 30
30 31
31 32
32
33 33 import os
34 34 from google.appengine.api import apiproxy_stub_map
35 35 from google.appengine.api import user_service_pb
…
63 63 A nickname is a human-readable string which uniquely identifies a Google
64 64 user, akin to a username. It will be an email address for some users, but
65 65 not all.
66
67 A user could be a Google Accounts user or a federated login user.
68
69 federated_identity and federated_provider are only avaliable for
70 federated users.
66 71 """
67 72
68 73
69 74 __user_id = None
70 75
71 def __init__(self, email=None, _auth_domain=None, _user_id=None):
76 def __init__(self, email=None, _auth_domain=None,
77 _user_id=None, federated_identity=None, federated_provider=None):
72 78 """Constructor.
73 79
74 80 Args:
75 81 email: An optional string of the user's email address. It defaults to
76 82 the current user's email address.
83 federated_identity: federated identity of user. It defaults to the current
84 user's federated identity.
85 federated_provider: federated provider url of user.
77 86
78 87 Raises:
79 UserNotFoundError: Raised if the user is not logged in and the email
80 argument is empty.
88 UserNotFoundError: Raised if the user is not logged in and both email
89 and federated identity are empty.
81 90 """
82 91 if _auth_domain is None:
83 92 _auth_domain = os.environ.get('AUTH_DOMAIN')
84 else:
85 assert email is not None
86
87 93 assert _auth_domain
88 94
89 if email is None:
90 assert 'USER_EMAIL' in os.environ
91 email = os.environ['USER_EMAIL']
92 if _user_id is None and 'USER_ID' in os.environ:
93 _user_id = os.environ['USER_ID']
95 if email is None and federated_identity is None:
96 email = os.environ.get('USER_EMAIL', email)
97 _user_id = os.environ.get('USER_ID', _user_id)
98 federated_identity = os.environ.get('FEDERATED_IDENTITY',
99 federated_identity)
100 federated_provider = os.environ.get('FEDERATED_PROVIDER',
101 federated_provider)
94 102
95 if not email:
103 if not email and not federated_identity:
96 104 raise UserNotFoundError
97 105
98 106 self.__email = email
107 self.__federated_identity = federated_identity
108 self.__federated_provider = federated_provider
99 109 self.__auth_domain = _auth_domain
100 110 self.__user_id = _user_id or None
101 111
…
104 114
105 115 The nickname will be a unique, human readable identifier for this user
106 116 with respect to this application. It will be an email address for some
107 users, but not all.
117 users, part of the email address for some users, and the federated identity
118 for federated users who have not asserted an email address.
108 119 """
109 120 if (self.__email and self.__auth_domain and
110 121 self.__email.endswith('@' + self.__auth_domain)):
111 122 suffix_len = len(self.__auth_domain) + 1
112 123 return self.__email[:-suffix_len]
124 elif self.__federated_identity:
125 return self.__federated_identity
113 126 else:
114 127 return self.__email
115 128
…
128 141 """Return this user's auth domain."""
129 142 return self.__auth_domain
130 143
144 def federated_identity(self):
145 """Return this user's federated identity, None if not a federated user."""
146 return self.__federated_identity
147
148 def federated_provider(self):
149 """Return this user's federated provider, None if not a federated user."""
150 return self.__federated_provider
151
131 152 def __unicode__(self):
132 153 return unicode(self.nickname())
133 154
…
135 156 return str(self.nickname())
136 157
137 158 def __repr__(self):
159 values = []
160 if self.__email:
161 values.append("email='%s'" % self.__email)
162 if self.__federated_identity:
163 values.append("federated_identity='%s'" % self.__federated_identity)
138 164 if self.__user_id:
139 return "users.User(email='%s',_user_id='%s')" % (self.email(),
140 self.user_id())
141 else:
142 return "users.User(email='%s')" % self.email()
165 values.append("_user_id='%s'" % self.__user_id)
166 return 'users.User(%s)' % ','.join(values)
143 167
144 168 def __hash__(self):
145 return hash((self.__email, self.__auth_domain))
169 if self.__federated_identity:
170 return hash((self.__federated_identity, self.__auth_domain))
171 else:
172 return hash((self.__email, self.__auth_domain))
146 173
147 174 def __cmp__(self, other):
148 175 if not isinstance(other, User):
149 176 return NotImplemented
150 return cmp((self.__email, self.__auth_domain),
151 (other.__email, other.__auth_domain))
177 if self.__federated_identity:
178 return cmp((self.__federated_identity, self.__auth_domain),
179 (other.__federated_identity, other.__auth_domain))
180 else:
181 return cmp((self.__email, self.__auth_domain),
182 (other.__email, other.__auth_domain))
152 183
153 184
154 def create_login_url(dest_url, _auth_domain=None):
155 """Computes the login URL for this request and specified destination URL.
185 def create_login_url(dest_url=None, _auth_domain=None,
186 federated_identity=None):
187 """Computes the login URL for redirection.
156 188
157 189 Args:
158 190 dest_url: String that is the desired final destination URL for the user
159 191 once login is complete. If 'dest_url' does not have a host
160 192 specified, we will use the host from the current request.
193 federated_identity: federated_identity is used to trigger OpenId Login
194 flow, an empty value will trigger Google OpenID Login
195 by default.
161 196
162 197 Returns:
163 string
198 Login URL as a string. If federated_identity is set, this will be
199 a federated login using the specified identity. If not, this
200 will use Google Accounts.
164 201 """
165 202 req = user_service_pb.CreateLoginURLRequest()
166 203 resp = user_service_pb.CreateLoginURLResponse()
167 204 req.set_destination_url(dest_url)
168 205 if _auth_domain:
169 206 req.set_auth_domain(_auth_domain)
207 if federated_identity:
208 req.set_federated_identity(federated_identity)
170 209
171 210 try:
172 211 apiproxy_stub_map.MakeSyncCall('user', 'CreateLoginURL', req, resp)
…
185 224
186 225
187 226 def create_logout_url(dest_url, _auth_domain=None):
188 """Computes the logout URL for this request and specified destination URL.
227 """Computes the logout URL for this request and specified destination URL,
228 for both federated login App and Google Accounts App.
189 229
190 230 Args:
191 231 dest_url: String that is the desired final destination URL for the user
…
193 233 specified, we will use the host from the current request.
194 234
195 235 Returns:
196 string
236 Logout URL as a string
197 237 """
198 238 req = user_service_pb.CreateLogoutURLRequest()
199 239 resp = user_service_pb.CreateLogoutURLResponse() 
diff -uNr google_appengine_1.3.3/google/appengine/datastore/entity_pb.py google_appengine_1.3.4_prerelease/google/appengine/datastore/entity_pb.py
google_appengine_1.3.3/google/appengine/datastore/entity_pb.py [2010-04-20 03:02:28.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/datastore/entity_pb.py [2010-05-13 06:32:30.000000000 +0900]
249 249 gaiaid_ = 0
250 250 has_obfuscated_gaiaid_ = 0
251 251 obfuscated_gaiaid_ = ""
252 has_federated_identity_ = 0
253 federated_identity_ = ""
254 has_federated_provider_ = 0
255 federated_provider_ = ""
252 256
253 257 def __init__(self, contents=None):
254 258 if contents is not None: self.MergeFromString(contents)
…
318 322
319 323 def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_
320 324
325 def federated_identity(self): return self.federated_identity_
326
327 def set_federated_identity(self, x):
328 self.has_federated_identity_ = 1
329 self.federated_identity_ = x
330
331 def clear_federated_identity(self):
332 if self.has_federated_identity_:
333 self.has_federated_identity_ = 0
334 self.federated_identity_ = ""
335
336 def has_federated_identity(self): return self.has_federated_identity_
337
338 def federated_provider(self): return self.federated_provider_
339
340 def set_federated_provider(self, x):
341 self.has_federated_provider_ = 1
342 self.federated_provider_ = x
343
344 def clear_federated_provider(self):
345 if self.has_federated_provider_:
346 self.has_federated_provider_ = 0
347 self.federated_provider_ = ""
348
349 def has_federated_provider(self): return self.has_federated_provider_
350
321 351
322 352 def MergeFrom(self, x):
323 353 assert x is not self
…
326 356 if (x.has_nickname()): self.set_nickname(x.nickname())
327 357 if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid())
328 358 if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid())
359 if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
360 if (x.has_federated_provider()): self.set_federated_provider(x.federated_provider())
329 361
330 362 def Equals(self, x):
331 363 if x is self: return 1
…
339 371 if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0
340 372 if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0
341 373 if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0
374 if self.has_federated_identity_ != x.has_federated_identity_: return 0
375 if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
376 if self.has_federated_provider_ != x.has_federated_provider_: return 0
377 if self.has_federated_provider_ and self.federated_provider_ != x.federated_provider_: return 0
342 378 return 1
343 379
344 380 def IsInitialized(self, debug_strs=None):
…
364 400 if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_))
365 401 n += self.lengthVarInt64(self.gaiaid_)
366 402 if (self.has_obfuscated_gaiaid_): n += 2 + self.lengthString(len(self.obfuscated_gaiaid_))
403 if (self.has_federated_identity_): n += 2 + self.lengthString(len(self.federated_identity_))
404 if (self.has_federated_provider_): n += 2 + self.lengthString(len(self.federated_provider_))
367 405 return n + 4
368 406
369 407 def Clear(self):
…
372 410 self.clear_nickname()
373 411 self.clear_gaiaid()
374 412 self.clear_obfuscated_gaiaid()
413 self.clear_federated_identity()
414 self.clear_federated_provider()
375 415
376 416 def OutputUnchecked(self, out):
377 417 out.putVarInt32(74)
…
386 426 if (self.has_obfuscated_gaiaid_):
387 427 out.putVarInt32(154)
388 428 out.putPrefixedString(self.obfuscated_gaiaid_)
429 if (self.has_federated_identity_):
430 out.putVarInt32(170)
431 out.putPrefixedString(self.federated_identity_)
432 if (self.has_federated_provider_):
433 out.putVarInt32(178)
434 out.putPrefixedString(self.federated_provider_)
389 435
390 436 def TryMerge(self, d):
391 437 while 1:
…
406 452 if tt == 154:
407 453 self.set_obfuscated_gaiaid(d.getPrefixedString())
408 454 continue
455 if tt == 170:
456 self.set_federated_identity(d.getPrefixedString())
457 continue
458 if tt == 178:
459 self.set_federated_provider(d.getPrefixedString())
460 continue
409 461 if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
410 462 d.skipData(tt)
411 463
…
417 469 if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_))
418 470 if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_))
419 471 if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_))
472 if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
473 if self.has_federated_provider_: res+=prefix+("federated_provider: %s\n" % self.DebugFormatString(self.federated_provider_))
420 474 return res
421 475
422 476 class PropertyValue_ReferenceValue(ProtocolBuffer.ProtocolMessage):
…
827 881 kUserValuenickname = 11
828 882 kUserValuegaiaid = 18
829 883 kUserValueobfuscated_gaiaid = 19
884 kUserValuefederated_identity = 21
885 kUserValuefederated_provider = 22
830 886 kReferenceValueGroup = 12
831 887 kReferenceValueapp = 13
832 888 kReferenceValuename_space = 20
…
857 913 18: "gaiaid",
858 914 19: "obfuscated_gaiaid",
859 915 20: "name_space",
860 }, 20)
916 21: "federated_identity",
917 22: "federated_provider",
918 }, 22)
861 919
862 920 _TYPES = _BuildTagLookupTable({
863 921 0: ProtocolBuffer.Encoder.NUMERIC,
…
881 939 18: ProtocolBuffer.Encoder.NUMERIC,
882 940 19: ProtocolBuffer.Encoder.STRING,
883 941 20: ProtocolBuffer.Encoder.STRING,
884 }, 20, ProtocolBuffer.Encoder.MAX_TYPE)
942 21: ProtocolBuffer.Encoder.STRING,
943 22: ProtocolBuffer.Encoder.STRING,
944 }, 22, ProtocolBuffer.Encoder.MAX_TYPE)
885 945
886 946 _STYLE = """"""
887 947 _STYLE_CONTENT_TYPE = """"""
…
1525 1585 gaiaid_ = 0
1526 1586 has_obfuscated_gaiaid_ = 0
1527 1587 obfuscated_gaiaid_ = ""
1588 has_federated_identity_ = 0
1589 federated_identity_ = ""
1590 has_federated_provider_ = 0
1591 federated_provider_ = ""
1528 1592
1529 1593 def __init__(self, contents=None):
1530 1594 if contents is not None: self.MergeFromString(contents)
…
1594 1658
1595 1659 def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_
1596 1660
1661 def federated_identity(self): return self.federated_identity_
1662
1663 def set_federated_identity(self, x):
1664 self.has_federated_identity_ = 1
1665 self.federated_identity_ = x
1666
1667 def clear_federated_identity(self):
1668 if self.has_federated_identity_:
1669 self.has_federated_identity_ = 0
1670 self.federated_identity_ = ""
1671
1672 def has_federated_identity(self): return self.has_federated_identity_
1673
1674 def federated_provider(self): return self.federated_provider_
1675
1676 def set_federated_provider(self, x):
1677 self.has_federated_provider_ = 1
1678 self.federated_provider_ = x
1679
1680 def clear_federated_provider(self):
1681 if self.has_federated_provider_:
1682 self.has_federated_provider_ = 0
1683 self.federated_provider_ = ""
1684
1685 def has_federated_provider(self): return self.has_federated_provider_
1686
1597 1687
1598 1688 def MergeFrom(self, x):
1599 1689 assert x is not self
…
1602 1692 if (x.has_nickname()): self.set_nickname(x.nickname())
1603 1693 if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid())
1604 1694 if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid())
1695 if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
1696 if (x.has_federated_provider()): self.set_federated_provider(x.federated_provider())
1605 1697
1606 1698 def Equals(self, x):
1607 1699 if x is self: return 1
…
1615 1707 if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0
1616 1708 if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0
1617 1709 if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0
1710 if self.has_federated_identity_ != x.has_federated_identity_: return 0
1711 if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
1712 if self.has_federated_provider_ != x.has_federated_provider_: return 0
1713 if self.has_federated_provider_ and self.federated_provider_ != x.federated_provider_: return 0
1618 1714 return 1
1619 1715
1620 1716 def IsInitialized(self, debug_strs=None):
…
1640 1736 if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_))
1641 1737 n += self.lengthVarInt64(self.gaiaid_)
1642 1738 if (self.has_obfuscated_gaiaid_): n += 1 + self.lengthString(len(self.obfuscated_gaiaid_))
1739 if (self.has_federated_identity_): n += 1 + self.lengthString(len(self.federated_identity_))
1740 if (self.has_federated_provider_): n += 1 + self.lengthString(len(self.federated_provider_))
1643 1741 return n + 3
1644 1742
1645 1743 def Clear(self):
…
1648 1746 self.clear_nickname()
1649 1747 self.clear_gaiaid()
1650 1748 self.clear_obfuscated_gaiaid()
1749 self.clear_federated_identity()
1750 self.clear_federated_provider()
1651 1751
1652 1752 def OutputUnchecked(self, out):
1653 1753 out.putVarInt32(10)
…
1662 1762 if (self.has_obfuscated_gaiaid_):
1663 1763 out.putVarInt32(42)
1664 1764 out.putPrefixedString(self.obfuscated_gaiaid_)
1765 if (self.has_federated_identity_):
1766 out.putVarInt32(50)
1767 out.putPrefixedString(self.federated_identity_)
1768 if (self.has_federated_provider_):
1769 out.putVarInt32(58)
1770 out.putPrefixedString(self.federated_provider_)
1665 1771
1666 1772 def TryMerge(self, d):
1667 1773 while d.avail() > 0:
…
1681 1787 if tt == 42:
1682 1788 self.set_obfuscated_gaiaid(d.getPrefixedString())
1683 1789 continue
1790 if tt == 50:
1791 self.set_federated_identity(d.getPrefixedString())
1792 continue
1793 if tt == 58:
1794 self.set_federated_provider(d.getPrefixedString())
1795 continue
1684 1796 if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
1685 1797 d.skipData(tt)
1686 1798
…
1692 1804 if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_))
1693 1805 if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_))
1694 1806 if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_))
1807 if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
1808 if self.has_federated_provider_: res+=prefix+("federated_provider: %s\n" % self.DebugFormatString(self.federated_provider_))
1695 1809 return res
1696 1810
1697 1811
…
1703 1817 knickname = 3
1704 1818 kgaiaid = 4
1705 1819 kobfuscated_gaiaid = 5
1820 kfederated_identity = 6
1821 kfederated_provider = 7
1706 1822
1707 1823 _TEXT = _BuildTagLookupTable({
1708 1824 0: "ErrorCode",
…
1711 1827 3: "nickname",
1712 1828 4: "gaiaid",
1713 1829 5: "obfuscated_gaiaid",
1714 }, 5)
1830 6: "federated_identity",
1831 7: "federated_provider",
1832 }, 7)
1715 1833
1716 1834 _TYPES = _BuildTagLookupTable({
1717 1835 0: ProtocolBuffer.Encoder.NUMERIC,
…
1720 1838 3: ProtocolBuffer.Encoder.STRING,
1721 1839 4: ProtocolBuffer.Encoder.NUMERIC,
1722 1840 5: ProtocolBuffer.Encoder.STRING,
1723 }, 5, ProtocolBuffer.Encoder.MAX_TYPE)
1841 6: ProtocolBuffer.Encoder.STRING,
1842 7: ProtocolBuffer.Encoder.STRING,
1843 }, 7, ProtocolBuffer.Encoder.MAX_TYPE)
1724 1844
1725 1845 _STYLE = """"""
1726 1846 _STYLE_CONTENT_TYPE = """""" 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_config.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_config.py
google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_config.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_config.py [2010-05-13 06:32:47.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Bulkloader Config Parser and runner.
19
20 A library to read bulkloader yaml configs.
21 The code to interface between the bulkloader tool and the various connectors
22 and conversions.
23 """
24
25
26
27
28 import copy
29 import os
30 import sys
31
32 from google.appengine.api import datastore
33 from google.appengine.ext.bulkload import bulkloader_errors
34 from google.appengine.ext.bulkload import bulkloader_parser
35 from google.appengine.ext.bulkload import csv_connector
36 from google.appengine.ext.bulkload import simpletext_connector
37 from google.appengine.ext.bulkload import simplexml_connector
38
39
40 CONNECTOR_FACTORIES = {
41 'csv': csv_connector.CsvConnector.create_from_options,
42 'simplexml': simplexml_connector.SimpleXmlConnector.create_from_options,
43 'simpletext': simpletext_connector.SimpleTextConnector.create_from_options,
44 }
45
46
47 class BulkloadState(object):
48 """Encapsulates state which is passed to other methods used in bulk loading.
49
50 It is optionally passed to import/export transform functions.
51 It is passed to connector objects.
52
53 Properties:
54 filename: The filename flag passed on the command line.
55 loader_opts: The loader_opts flag passed on the command line.
56 exporter_opts: The exporter_opts flag passed on the command line.
57 current_instance: The current entity or model instance.
58 current_entity: On export, the current entity instance.
59 current_dictionary: The current input or output dictionary.
60 """
61
62 def __init__(self):
63 self.filename = ''
64 self.loader_opts = None
65 self.exporter_opts = None
66 self.current_instance = None
67 self.current_entity = None
68 self.current_dictionary = None
69
70
71 def default_export_transform(value):
72 """A default export transform if nothing else is specified.
73
74 We assume most export connectors are string based, so a string cast is used.
75 However, casting None to a string leads to 'None', so that's special cased.
76
77 Args:
78 value: A value of some type.
79
80 Returns:
81 unicode(value), or u'' if value is None
82 """
83 if value is None:
84 return u''
85 else:
86 return unicode(value)
87
88
89 class DictConvertor(object):
90 """Convert a dict to an App Engine model instance or entity. And back.
91
92 The constructor takes a transformer spec representing a single transformer
93 in a bulkloader.yaml.
94
95 The DictConvertor object has two public methods, dict_to_entity and
96 entity_to_dict, which do the conversion between a neutral dictionary (the
97 input/output of a connector) and an entity based on the spec.
98
99 Note that the model class may be used instead of an entity during the
100 transform--this adds extra validation, etc, but also has a performance hit.
101 """
102
103 def __init__(self, transformer_spec):
104 """Constructor. See class docstring for more info.
105
106 Args:
107 transformer_spec: A single transformer from a parsed bulkloader.yaml.
108 This assumes that the transformer_spec is valid. It does not
109 double check things like use_model_on_export requiring model.
110 """
111 self._transformer_spec = transformer_spec
112
113 self._create_key = None
114 for prop in self._transformer_spec.property_map:
115 if prop.property == '__key__':
116 self._create_key = prop
117
118 def dict_to_entity(self, input_dict, bulkload_state):
119 """Transform the dict to a model or entity instance(s).
120
121 Args:
122 input_dict: Neutral input dictionary describing a single input record.
123 bulkload_state: bulkload_state object describing the state.
124
125 Returns:
126 Entity or model instance, or collection of entity or model instances,
127 to be uploaded.
128 """
129 bulkload_state_copy = copy.copy(bulkload_state)
130 bulkload_state_copy.current_dictionary = input_dict
131 instance = self.__create_instance(input_dict, bulkload_state_copy)
132 bulkload_state_copy.current_instance = instance
133 self.__run_import_transforms(input_dict, instance, bulkload_state_copy)
134 if self._transformer_spec.post_import_function:
135 post_map_instance = self._transformer_spec.post_import_function(
136 input_dict, instance, bulkload_state_copy)
137 return post_map_instance
138 return instance
139
140 def entity_to_dict(self, entity, bulkload_state):
141 """Transform the entity to a dict, possibly via a model.
142
143 Args:
144 entity: An entity.
145 bulkload_state: bulkload_state object describing the global state.
146
147 Returns:
148 A neutral output dictionary describing the record to write to the
149 output.
150 In the future this may return zero or multiple output dictionaries.
151 """
152 if self._transformer_spec.use_model_on_export:
153 instance = self._transformer_spec.model.from_entity(entity)
154 else:
155 instance = entity
156
157 export_dict = {}
158 bulkload_state.current_entity = entity
159 bulkload_state.current_instance = instance
160 bulkload_state.current_dictionary = export_dict
161 self.__run_export_transforms(instance, export_dict, bulkload_state)
162 if self._transformer_spec.post_export_function:
163 post_export_result = self._transformer_spec.post_export_function(
164 instance, export_dict, bulkload_state)
165 return post_export_result
166 return export_dict
167
168 def __dict_to_prop(self, transform, input_dict, bulkload_state):
169 """Handle a single property on import.
170
171 Args:
172 transform: The transform spec for this property.
173 input_dict: Neutral input dictionary describing a single input record.
174 bulkload_state: bulkload_state object describing the global state.
175
176 Returns:
177 The value for this particular property.
178 """
179 if transform.import_template:
180 value = transform.import_template % input_dict
181 else:
182 value = input_dict.get(transform.external_name)
183
184 if transform.import_transform:
185 if transform.import_transform.supports_bulkload_state:
186 value = transform.import_transform(value, bulkload_state=bulkload_state)
187 else:
188 value = transform.import_transform(value)
189 return value
190
191 def __create_instance(self, input_dict, bulkload_state):
192 """Return a model instance or entity from an input_dict.
193
194 Args:
195 input_dict: Neutral input dictionary describing a single input record.
196 bulkload_state: bulkload_state object describing the global state.
197
198 Returns:
199 Entity or model instance, or collection of entity or model instances,
200 to be uploaded.
201 """
202 key = None
203 parent = None
204 if self._create_key:
205 key = self.__dict_to_prop(self._create_key, input_dict, bulkload_state)
206 if isinstance(key, datastore.Key):
207 if not key.name():
208 raise bulkloader_errors.ErrorOnTransform(
209 'Numeric keys are not supported on input at this time.')
210 parent = key.parent()
211 key = key.name()
212
213 if self._transformer_spec.model:
214 instance = self._transformer_spec.model(key_name=key, parent=parent)
215 else:
216 instance = datastore.Entity(self._transformer_spec.kind,
217 parent=parent, name=key)
218 return instance
219
220 def __run_import_transforms(self, input_dict, instance, bulkload_state):
221 """Fill in a single entity or model instance from an input_dict.
222
223 Args:
224 input_dict: Input dict from the connector object.
225 instance: Entity or model instance to fill in.
226 bulkload_state: Passed bulkload state.
227 """
228
229 for transform in self._transformer_spec.property_map:
230 if transform.property == '__key__':
231 continue
232
233 value = self.__dict_to_prop(transform, input_dict, bulkload_state)
234 if self._transformer_spec.model:
235 setattr(instance, transform.property, value)
236 else:
237 instance[transform.property] = value
238
239 def __prop_to_dict(self, value, property_name, transform, export_dict,
240 bulkload_state):
241 """Transform a single export-side field value to dict property.
242
243 Args:
244 value: Value from the entity or model instance.
245 property_name: Name of the value in the entity or model instance.
246 transform: Transform property, either an ExportEntry or PropertyEntry
247 export_dict: output dictionary.
248 bulkload_state: Passed bulkload state.
249
250 Raises:
251 ErrorOnTransform, encapsulating an error encountered during the transform.
252 """
253 if transform.export_transform:
254 try:
255 if transform.export_transform.supports_bulkload_state:
256 transformed_value = transform.export_transform(
257 value, bulkload_state=bulkload_state)
258 else:
259 transformed_value = transform.export_transform(value)
260 except Exception, err:
261 raise bulkloader_errors.ErrorOnTransform(
262 'Error on transform. '
263 'Property: %s External Name: %s. Code: %s Details: %s' %
264 (property_name, transform.external_name, transform.export_transform,
265 err))
266 else:
267 transformed_value = default_export_transform(value)
268 export_dict[transform.external_name] = transformed_value
269
270 def __run_export_transforms(self, instance, export_dict, bulkload_state):
271 """Fill in export_dict for an entity or model instance.
272
273 Args:
274 instance: Entity or model instance
275 export_dict: output dictionary.
276 bulkload_state: Passed bulkload state.
277 """
278 for transform in self._transformer_spec.property_map:
279 if transform.property == '__key__':
280 value = instance.key()
281 elif self._transformer_spec.use_model_on_export:
282 value = getattr(instance, transform.property, transform.default_value)
283 else:
284 value = instance.get(transform.property, transform.default_value)
285
286 if transform.export:
287 for prop in transform.export:
288 self.__prop_to_dict(value, transform.property, prop, export_dict,
289 bulkload_state)
290 elif transform.external_name:
291 self.__prop_to_dict(value, transform.property, transform, export_dict,
292 bulkload_state)
293
294
295 class GenericImporter(object):
296 """Generic Bulkloader import class for input->dict->model transformation.
297
298 The bulkloader will call generate_records and create_entity, and
299 we'll delegate those to the passed in methods.
300 """
301
302 def __init__(self, import_record_iterator, dict_to_entity, name):
303 """Constructor.
304
305 Args:
306 import_record_iterator: Method which yields neutral dictionaries.
307 dict_to_entity: Method dict_to_entity(input_dict) returns model or entity
308 instance(s).
309 name: Name to register with the bulkloader importers (as 'kind').
310 """
311 self.import_record_iterator = import_record_iterator
312 self.dict_to_entity = dict_to_entity
313 self.kind = name
314 self.bulkload_state = BulkloadState()
315
316 def get_high_ids(self):
317 """Required as part of the bulkloader Loader interface.
318
319 At the moment, this is not actually used by the bulkloader for import, as
320 import does not currently support specifying numeric ids for keys.
321 (Unspecified keys will become autogenerated ids.)
322
323 Returns:
324 dict {ancestor_path : {kind : id}} of high id values, curently always {}.
325 """
326 return {}
327
328 def initialize(self, filename, loader_opts):
329 """Performs initialization. Merely records the values for later use.
330
331 Args:
332 filename: The string given as the --filename flag argument.
333 loader_opts: The string given as the --loader_opts flag argument.
334 """
335
336 self.bulkload_state.loader_opts = loader_opts
337 self.bulkload_state.filename = filename
338
339 def finalize(self):
340 """Performs finalization actions after the upload completes."""
341 pass
342
343 def generate_records(self, filename):
344 """Iterator yielding neutral dictionaries from the connector object.
345
346 Args:
347 filename: Filename argument passed in on the command line.
348
349 Returns:
350 Iterator yielding neutral dictionaries, later passed to create_entity.
351 """
352 return self.import_record_iterator(filename, self.bulkload_state)
353
354 def generate_key(self, line_number, unused_values):
355 """Bulkloader method to generate keys, mostly unused here.
356
357 This is called by the bulkloader just before it calls create_entity. The
358 line_number is returned to be passed to the record dict, but otherwise
359 unused.
360
361 Args:
362 line_number: Record number from the bulkloader.
363 unused_values: Neutral dict from generate_records; unused.
364
365 Returns:
366 line_number for use later on.
367 """
368 return line_number
369
370 def create_entity(self, values, key_name=None, parent=None):
371 """Creates entity/entities from input values via the dict_to_entity method.
372
373 Args:
374 values: Neutral dict from generate_records.
375 key_name: record number from generate_key.
376 parent: Always None in this implementation of a Loader.
377
378 Returns:
379 Entity or model instance, or collection of entity or model instances,
380 to be uploaded.
381 """
382
383 input_dict = values
384 input_dict['__record_number__'] = key_name
385 return self.dict_to_entity(input_dict, self.bulkload_state)
386
387
388 class GenericExporter(object):
389 """Implements bulkloader.Exporter interface and delegates.
390
391 This will delegate to the passed in entity_to_dict method and the
392 methods on the export_recorder which are in the ConnectorInterface.
393 """
394
395 def __init__(self, export_recorder, entity_to_dict, kind,
396 sort_key_from_entity):
397 """Constructor.
398
399 Args:
400 export_recorder: Object which writes results, an implementation of
401 ConnectorInterface.
402 entity_to_dict: Method which converts a single entity to a neutral dict.
403 kind: Kind to identify this object to the bulkloader.
404 sort_key_from_entity: Optional method to return a sort key for each
405 entity. This key will be used to sort the downloaded entities before
406 passing them to eneity_to_dict.
407 """
408 self.export_recorder = export_recorder
409 self.entity_to_dict = entity_to_dict
410 self.kind = kind
411 self.sort_key_from_entity = sort_key_from_entity
412 self.calculate_sort_key_from_entity = bool(sort_key_from_entity)
413 self.bulkload_state = BulkloadState()
414
415 def initialize(self, filename, exporter_opts):
416 """Performs initialization and validation of the output file.
417
418 Args:
419 filename: The string given as the --filename flag argument.
420 exporter_opts: The string given as the --exporter_opts flag argument.
421 """
422 self.bulkload_state.filename = filename
423 self.bulkload_state.exporter_opts = exporter_opts
424 self.export_recorder.initialize_export(filename, self.bulkload_state)
425
426 def output_entities(self, entity_iterator):
427 """Outputs the downloaded entities.
428
429 Args:
430 entity_iterator: An iterator that yields the downloaded entities
431 in sorted order.
432 """
433 for entity in entity_iterator:
434 output_dict = self.entity_to_dict(entity, self.bulkload_state)
435 if output_dict:
436 self.export_recorder.write_dict(output_dict)
437
438 def finalize(self):
439 """Performs finalization actions after the download completes."""
440 self.export_recorder.finalize_export()
441
442
443 def create_transformer_classes(transformer_spec, config_globals):
444 """Create an importer and exporter class from a transformer spec.
445
446 Args:
447 transformer_spec: A bulkloader_parser.TransformerEntry.
448 config_globals: Dict to use to reference globals for code in the config.
449
450 Raises:
451 InvalidConfig: when the config is invalid.
452
453 Returns:
454 Tuple, (importer class, exporter class), each which is in turn a wrapper
455 for the GenericImporter/GenericExporter class using a DictConvertor object
456 configured as per the transformer_spec.
457 """
458 if transformer_spec.connector in CONNECTOR_FACTORIES:
459 connector_factory = CONNECTOR_FACTORIES[transformer_spec.connector]
460 elif config_globals and '.' in transformer_spec.connector:
461 try:
462 connector_factory = eval(transformer_spec.connector, config_globals)
463 except (NameError, AttributeError):
464 raise bulkloader_errors.InvalidConfiguration(
465 'Invalid connector specified for name=%s. Could not evaluate %s.' %
466 (transformer_spec.name, transformer_spec.connector))
467 else:
468 raise bulkloader_errors.InvalidConfiguration(
469 'Invalid connector specified for name=%s. Must be either a built in '
470 'connector ("%s") or a factory method in a module imported via '
471 'python_preamble.' %
472 (transformer_spec.name, '", "'.join(CONNECTOR_FACTORIES)))
473 options = {}
474 if transformer_spec.connector_options:
475 options = transformer_spec.connector_options.ToDict()
476
477 try:
478 connector_object = connector_factory(options, transformer_spec.name)
479 except TypeError:
480 raise bulkloader_errors.InvalidConfiguration(
481 'Invalid connector specified for name=%s. Could not initialize %s.' %
482 (transformer_spec.name, transformer_spec.connector))
483
484
485 dict_to_model_object = DictConvertor(transformer_spec)
486
487 class ImporterClass(GenericImporter):
488 """Class to pass to the bulkloader, wraps the specificed configuration."""
489
490 def __init__(self):
491 super(self.__class__, self).__init__(
492 connector_object.generate_import_record,
493 dict_to_model_object.dict_to_entity,
494 transformer_spec.name)
495 importer_class = ImporterClass
496
497 class ExporterClass(GenericExporter):
498 """Class to pass to the bulkloader, wraps the specificed configuration."""
499
500 def __init__(self):
501 super(self.__class__, self).__init__(
502 connector_object,
503 dict_to_model_object.entity_to_dict,
504 transformer_spec.kind,
505 transformer_spec.sort_key_from_entity)
506 exporter_class = ExporterClass
507
508 return importer_class, exporter_class
509
510
511 def load_config_from_stream(stream):
512 """Parse a bulkloader.yaml file into bulkloader loader classes.
513
514 Args:
515 stream: A stream containing bulkloader.yaml data.
516
517 Returns:
518 importer_classes, exporter_classes: Constructors suitable to pass to the
519 bulkloader.
520 """
521 config_globals = {}
522 config = bulkloader_parser.load_config(stream, config_globals)
523 importer_classes = []
524 exporter_classes = []
525 for transformer in config.transformers:
526 importer, exporter = create_transformer_classes(transformer, config_globals)
527 if importer:
528 importer_classes.append(importer)
529 if exporter:
530 exporter_classes.append(exporter)
531
532 return importer_classes, exporter_classes
533
534
535 def load_config(filename, update_path=True):
536 """Load a configuration file and create importer and exporter classes.
537
538 Args:
539 filename: Filename of bulkloader.yaml.
540 update_path: Should sys.path be extended to include the path of filename?
541
542 Returns:
543 Tuple, (importer classes, exporter classes) based on the transformers
544 specified in the file.
545 """
546
547 if update_path:
548 sys.path.append(os.path.abspath(os.path.dirname(os.path.abspath(filename))))
549 stream = file(filename, 'r')
550 try:
551 return load_config_from_stream(stream)
552 finally:
553 stream.close() 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_errors.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_errors.py
google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_errors.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_errors.py [2010-05-13 06:32:48.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Exceptions raised by bulkloader methods."""
19
20
21
22
23 class Error(Exception):
24 """Base bulkloader error type."""
25
26
27 class ErrorOnTransform(Error):
28 """An exception was raised during this transform."""
29
30
31 class InvalidConfiguration(Error):
32 """The configuration is invalid."""
33
34
35 class InvalidCodeInConfiguration(Error):
36 """A code or lambda statement in the configuration could not be evaulated."""
37
38
39 class InvalidExportData(Error):
40 """The export data cannot be written using this connector object.""" 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_parser.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_parser.py
google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_parser.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_parser.py [2010-05-13 06:32:48.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Bulkloader Config Parser and runner.
19
20 A library to read bulkloader yaml configs. Returns a BulkloaderEntry object
21 which describes the bulkloader.yaml in object form, including some additional
22 parsing of things like Python lambdas.
23 """
24
25
26
27
28 import inspect
29 import sys
30
31 from google.appengine.api import validation
32 from google.appengine.api import yaml_builder
33 from google.appengine.api import yaml_listener
34 from google.appengine.api import yaml_object
35
36 from google.appengine.ext.bulkload import bulkloader_errors
37
38
39 _global_temp_globals = None
40
41
42 class EvaluatedCallable(validation.Validator):
43 """Validates that a string evaluates to a Python callable.
44
45 Calls eval at validation time and stores the results as a ParsedMethod object.
46 The ParsedMethod object can be used as a string (original value) or callable
47 (parsed method). It also exposes supports_bulkload_state if the callable has
48 a kwarg called 'bulkload_state', which is used to determine how to call
49 the *_transform methods.
50 """
51
52 class ParsedMethod(object):
53 """Wrap the string, the eval'd method, and supports_bulkload_state."""
54
55 def __init__(self, value, key):
56 """Initialze internal state.
57
58 Eval the string value and save the result.
59
60 Args:
61 value: String to compile as a regular expression.
62 key: The YAML field name.
63
64 Raises:
65 InvalidCodeInConfiguration: if the code could not be evaluated, or
66 the evalauted method is not callable.
67 """
68 self.value = value
69 try:
70 self.method = eval(value, _global_temp_globals)
71 except Exception, err:
72 raise bulkloader_errors.InvalidCodeInConfiguration(
73 'Invalid code for %s. Code: "%s". Details: %s' % (key, value, err))
74 if not callable(self.method):
75 raise bulkloader_errors.InvalidCodeInConfiguration(
76 'Code for %s did not return a callable. Code: "%s".' %
77 (key, value))
78
79 self.supports_bulkload_state = False
80 try:
81 argspec = inspect.getargspec(self.method)
82 if 'bulkload_state' in argspec[0]:
83 self.supports_bulkload_state = True
84 except TypeError:
85 pass
86
87 def __str__(self):
88 """Return a string representation of the method: the original string."""
89 return self.value
90
91 def __call__(self, *args, **kwargs):
92 """Call the method."""
93 return self.method(*args, **kwargs)
94
95 def __init__(self):
96 """Initialize EvaluatedCallable validator."""
97 super(EvaluatedCallable, self).__init__()
98
99 def Validate(self, value, key):
100 """Validates that the string compiles as a Python callable.
101
102 Args:
103 value: String to compile as a regular expression.
104 key: The YAML field name.
105
106 Returns:
107 Value wrapped in an object with properties 'value' and 'fn'.
108
109 Raises:
110 InvalidCodeInConfiguration when value does not compile.
111 """
112 if isinstance(value, self.ParsedMethod):
113 return value
114 else:
115 return self.ParsedMethod(value, key)
116
117 def ToValue(self, value):
118 """Returns the code string for this value."""
119 return value.value
120
121
122 OPTIONAL_EVALUATED_CALLABLE = validation.Optional(EvaluatedCallable())
123
124
125 class ConnectorSubOptions(validation.Validated):
126 """Connector options."""
127
128 ATTRIBUTES = {
129 'delimiter': validation.Optional(validation.TYPE_STR),
130 'dialect': validation.Optional(validation.TYPE_STR),
131 }
132
133
134 class ConnectorOptions(validation.Validated):
135 """Connector options."""
136
137 ATTRIBUTES = {
138 'column_list':
139 validation.Optional(validation.Repeated(validation.TYPE_STR)),
140 'columns': validation.Optional(validation.TYPE_STR),
141 'encoding': validation.Optional(validation.TYPE_STR),
142 'epilog': validation.Optional(validation.TYPE_STR),
143 'export_options': validation.Optional(ConnectorSubOptions),
144 'import_options': validation.Optional(ConnectorSubOptions),
145 'mode': validation.Optional(validation.TYPE_STR),
146 'prolog': validation.Optional(validation.TYPE_STR),
147 'style': validation.Optional(validation.TYPE_STR),
148 'template': validation.Optional(validation.TYPE_STR),
149 'xpath_to_nodes': validation.Optional(validation.TYPE_STR),
150 'print_export_header_row': validation.Optional(validation.TYPE_BOOL),
151 'skip_import_header_row': validation.Optional(validation.TYPE_BOOL),
152 }
153
154 def CheckInitialized(self):
155 """Post-loading 'validation'. Really used to fix up yaml hackyness."""
156 super(ConnectorOptions, self).CheckInitialized()
157
158 if self.column_list:
159 self.column_list = [str(column) for column in self.column_list]
160
161
162 class ExportEntry(validation.Validated):
163 """Describes the optional export transform for a single property."""
164
165 ATTRIBUTES = {
166 'external_name': validation.Optional(validation.TYPE_STR),
167 'export_transform': OPTIONAL_EVALUATED_CALLABLE,
168 }
169
170
171 class PropertyEntry(validation.Validated):
172 """Describes the transform for a single property."""
173
174 ATTRIBUTES = {
175 'property': validation.Type(str),
176 'import_transform': OPTIONAL_EVALUATED_CALLABLE,
177 'import_template': validation.Optional(validation.TYPE_STR),
178 'default_value': validation.Optional(validation.TYPE_STR),
179 'export': validation.Optional(validation.Repeated(ExportEntry)),
180 }
181 ATTRIBUTES.update(ExportEntry.ATTRIBUTES)
182
183 def CheckInitialized(self):
184 """Check that all required (combinations) of fields are set.
185
186 Also fills in computed properties.
187
188 Raises:
189 InvalidConfiguration: If the config is invalid.
190 """
191 super(PropertyEntry, self).CheckInitialized()
192
193 if not (self.external_name or self.import_template or self.export):
194 raise bulkloader_errors.InvalidConfiguration(
195 'Neither external_name nor import_template nor export specified for '
196 'property %s.' % self.property)
197
198
199 class TransformerEntry(validation.Validated):
200 """Describes the transform for an entity (or model) kind."""
201
202 ATTRIBUTES = {
203 'name': validation.Optional(validation.TYPE_STR),
204 'kind': validation.Optional(validation.TYPE_STR),
205 'model': OPTIONAL_EVALUATED_CALLABLE,
206 'connector': validation.TYPE_STR,
207 'connector_options': validation.Optional(ConnectorOptions, {}),
208 'use_model_on_export': validation.Optional(validation.TYPE_BOOL),
209 'sort_key_from_entity': OPTIONAL_EVALUATED_CALLABLE,
210 'post_import_function': OPTIONAL_EVALUATED_CALLABLE,
211 'post_export_function': OPTIONAL_EVALUATED_CALLABLE,
212 'property_map': validation.Repeated(PropertyEntry, default=[]),
213 }
214
215 def CheckInitialized(self):
216 """Check that all required (combinations) of fields are set.
217
218 Also fills in computed properties.
219
220 Raises:
221 InvalidConfiguration: if the config is invalid.
222 """
223 if not self.kind and not self.model:
224 raise bulkloader_errors.InvalidConfiguration(
225 'Neither kind nor model specified for transformer.')
226 if self.kind and self.model:
227 raise bulkloader_errors.InvalidConfiguration(
228 'Both kind and model specified for transformer.')
229
230 if self.model:
231 self.kind = self.model().kind()
232 else:
233 if self.use_model_on_export:
234 raise bulkloader_errors.InvalidConfiguration(
235 'No model class specified but use_model_on_export is true.')
236 if not self.name:
237 self.name = self.kind
238
239 if not self.connector:
240 raise bulkloader_errors.InvalidConfiguration('No connector specified.')
241
242 property_names = set()
243 for prop in self.property_map:
244 if prop.property in property_names:
245 raise bulkloader_errors.InvalidConfiguration(
246 'Duplicate property specified for property %s in transform %s' %
247 (prop.property, self.name))
248 property_names.add(prop.property)
249
250
251 class PythonPreambleEntry(validation.Validated):
252 """Python modules to import at initialization time, typically models."""
253
254 ATTRIBUTES = {'import': validation.TYPE_STR,
255 'as': validation.Optional(validation.TYPE_STR),
256 }
257
258 def CheckInitialized(self):
259 """Check that all required fields are set, and update global state.
260
261 The imports specified in the preamble are imported at this time.
262 """
263 python_import = getattr(self, 'import')
264 topname = python_import.split('.')[0]
265 module_name = getattr(self, 'as')
266 if not module_name:
267 module_name = python_import.split('.')[-1]
268
269 __import__(python_import, _global_temp_globals)
270 _global_temp_globals[topname] = sys.modules[topname]
271 _global_temp_globals[module_name] = sys.modules[python_import]
272
273
274 class BulkloaderEntry(validation.Validated):
275 """Root of the bulkloader configuration."""
276
277 ATTRIBUTES = {
278 'python_preamble':
279 validation.Optional(validation.Repeated(PythonPreambleEntry)),
280 'transformers': validation.Repeated(TransformerEntry),
281 }
282
283
284 def load_config(stream, config_globals):
285 """Load a configuration file and generate importer and exporter classes.
286
287 Args:
288 stream: Stream containing config YAML.
289 config_globals: Dict to use to reference globals for code in the config.
290
291 Returns:
292 BulkloaderEntry
293
294 Raises:
295 InvalidConfiguration: If the config is invalid.
296 """
297 builder = yaml_object.ObjectBuilder(BulkloaderEntry)
298 handler = yaml_builder.BuilderHandler(builder)
299 listener = yaml_listener.EventListener(handler)
300 global _global_temp_globals
301 _global_temp_globals = config_globals
302 try:
303 listener.Parse(stream)
304 finally:
305 _global_temp_globals = None
306
307 bulkloader_infos = handler.GetResults()
308 if len(bulkloader_infos) < 1:
309 raise bulkloader_errors.InvalidConfiguration('No configuration specified.')
310 if len(bulkloader_infos) > 1:
311 raise bulkloader_errors.InvalidConfiguration(
312 'Multiple sections in configuration.')
313 bulkloader_info = bulkloader_infos[0]
314 if not bulkloader_info.transformers:
315 raise bulkloader_errors.InvalidConfiguration('No transformers specified.')
316 return bulkloader_info 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.py
google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.py [2010-05-13 06:32:48.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Wizard to generate bulkloader configuration.
19
20 Helper functions to call from the bulkloader.yaml.
21 The wizard is run by having bulkloader.py download datastore statistics
22 (http://code.google.com/appengine/docs/python/datastore/stats.html ,
23 specifically __Stat_PropertyType_PropertyName_Kind__) configured with
24 bulkloader_wizard.yaml.
25 """
26
27
28 PROPERTY_DUPE_WARNING = (
29 ' # Warning: This property is a duplicate, but with a different type.\n'
30 ' # TODO: Edit this transform so only one property with this name '
31 'remains.\n')
32
33 KIND_PREAMBLE = """
34 - kind: %(kind_name)s
35 connector: # TODO: Choose a connector here: csv, simplexml, etc...
36 connector_options:
37 # TODO: Add connector options here--these are specific to each connector.
38 property_map:
39 - property: __key__
40 external_name: key
41 export_transform: transform.key_id_or_name_as_string
42
43 """
44
45
46 class StatPostTransform(object):
47 """Create text to insert between properties and filter out 'bad' properties.
48
49 This class is a callable post_export_function which saves state
50 across multiple calls.
51
52 It uses this saved state to determine if each entity is the first entity seen
53 of a new kind, a duplicate kind/propertyname entry, or just a new property
54 in the current kind being processed.
55
56 It will suppress bad output by returning None for NULL property types and
57 __private__ types (notably the stats themselves).
58 """
59
60 def __init__(self):
61 """Constructor.
62
63 Attributes:
64 seen_properties: (kind, propertyname) -> number of times seen before. If
65 seen more than once, this is a duplicate property for the kind.
66 last_seen: Previous kind seen. If it changes, this is a new kind.
67 """
68 self.seen_properties = {}
69 self.last_seen = None
70
71 def __call__(self, instance, dictionary, bulkload_state):
72 """Implementation of StatPropertyTypePropertyNameKindPostExport.
73
74 See class docstring for more info.
75
76 Args:
77 instance: Input, current entity being exported.
78 dictionary: Output, dictionary created by property_map transforms.
79 bulkload_state: Passed bulkload_state.
80
81 Returns:
82 Dictionary--same object as passed in dictionary.
83 """
84 kind_name = dictionary['kind_name']
85 property_name = dictionary['property_name']
86 property_type = dictionary['property_type']
87
88 if kind_name.startswith('__'):
89 return None
90 if property_type == 'NULL':
91 return None
92
93 property_key = kind_name, property_name
94 if kind_name != self.last_seen:
95 self.last_seen = kind_name
96 separator = KIND_PREAMBLE % dictionary
97 elif property_key in self.seen_properties:
98 separator = PROPERTY_DUPE_WARNING % dictionary
99 else:
100 separator = ''
101 self.seen_properties[property_key] = (
102 self.seen_properties.get(property_key, 0) + 1)
103
104 dictionary['separator'] = separator
105 return dictionary
106
107
108 TYPE_TO_TRANSFORM_MAP = {
109 'Blob': ('transform.blobproperty_from_base64',
110 'base64.b64encode'),
111 'Boolean': ('transform.regexp_bool(\'true\', re.IGNORECASE)',
112 None),
113 'ByteString': ('transform.bytestring_from_base64', 'base64.b64encode'),
114 'Category': ('db.Category', None),
115 'Date/Time': ('transform.import_date_time(\'%Y-%m-%dT%H:%M:%S\')',
116 'transform.export_date_time(\'%Y-%m-%dT%H:%M:%S\')'),
117 'Email': ('db.Email', None),
118 'Float': ('transform.none_if_empty(float)', None),
119
120 'Integer': ('transform.none_if_empty(int)', None),
121 'Key': ('transform.create_foreign_key(\'TODO: fill in Kind name\')',
122 'transform.key_id_or_name_as_string'),
123 'Link': ('db.Link', None),
124
125 'PhoneNumber': ('db.PhoneNumber', None),
126 'PostalAddress': ('db.PostalAddress', None),
127 'Rating': ('transform.none_if_empty(db.Rating)', None),
128 'String': (None, None),
129 'Text': ('db.Text', None),
130 'User': ('transform.none_if_empty(users.User) # Assumes email address',
131 None),
132 }
133
134
135 def DatastoreTypeToTransforms(property_type):
136 """Return the import/export_transform lines for a datastore type.
137
138 Args:
139 property_type: Property type from the KindPropertyNamePropertyTypeStat.
140
141 Returns:
142 Strings for use in a bulkloader.yaml as transforms. This
143 may be '' (no transform needed), or one or two lines with import_transform
144 or export_transform.
145 """
146 import_transform, export_transform = TYPE_TO_TRANSFORM_MAP.get(property_type,
147 (None, None))
148 transform = []
149 if import_transform:
150 transform.append(' import_transform: %s\n' % import_transform)
151 if export_transform:
152 transform.append(' export_transform: %s\n' % export_transform)
153
154 return ''.join(transform) 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.yaml google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.yaml
google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.yaml [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.yaml [2010-05-13 06:32:47.000000000 +0900]
1 # Transform description for generating bulkloader configuration by bulkloading
2 # the datastore stats.
3
4 python_preamble:
5 - import: base64
6 - import: re
7 - import: google.appengine.ext.bulkload.transform
8 - import: google.appengine.ext.bulkload.bulkloader_wizard
9 - import: google.appengine.api.datastore
10 - import: google.appengine.api.users
11
12 transformers:
13 - kind: __Stat_PropertyType_PropertyName_Kind__
14 connector: simpletext
15 connector_options:
16 mode: nonewline
17 prolog: |
18 # Autogenerated bulkloader.yaml file.
19 # You're likely to have to do various edits to this file:
20 # At a minimum address the items marked with TODO:
21 # * Fill in connector and connector_options
22 # * Review the property_map.
23 # - Ensure the 'external_name' matches the name of your CSV column,
24 # XML tag, etc.
25 # - Check that __key__ property is what you want. Its value will become
26 # the key name on import, and on export the value will be the Key
27 # object. If you would like automatic key generation on import and
28 # omitting the key on export, you can remove the entire __key__
29 # property from the property map.
30
31 # If you have module(s) with your model classes, add them here. Also
32 # change the kind properties to model_class.
33 python_preamble:
34 - import: google.appengine.ext.bulkload.transform
35 - import: google.appengine.ext.db
36 - import: re
37 - import: base64
38
39 transformers:
40 template: |
41 %(separator)s - property: %(property_name)s
42 external_name: %(property_name)s
43 # Type: %(property_type)s Stats: %(count)s properties of this type in this kind.
44 %(transforms)s
45 sort_key_from_entity:
46 "lambda entity: '_'.join((entity.get('kind_name', ''),
47 entity.get('property_name', ''),
48 entity.get('property_type', '')))"
49 property_map:
50 - property: __key__
51 external_name: key
52 export_transform: datastore.Key.name
53 - property: kind_name
54 external_name: kind_name
55 default_value: MISSING_KIND_NAME
56 - property: property_name
57 external_name: property_name
58 - property: property_type
59 external_name: property_type
60 export:
61 - external_name: property_type
62 - external_name: transforms
63 export_transform: bulkloader_wizard.DatastoreTypeToTransforms
64 - property: bytes
65 external_name: bytes
66 - property: count
67 external_name: count
68 - property: timestamp
69 external_name: timestamp
70 export_transform: transform.export_date_time('%Y%m%dT%H:%M')
71 post_export_function: bulkloader_wizard.StatPostTransform() 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/connector_interface.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/connector_interface.py
google_appengine_1.3.3/google/appengine/ext/bulkload/connector_interface.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/connector_interface.py [2010-05-13 06:32:47.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Bulkloader interfaces for the format reader/writers."""
19
20
21
22
23
24 class ConnectorInterface(object):
25 """Abstract base class describing the external Connector interface.
26
27 The External Connector interface describes the transition between an external
28 data source, e.g. CSV file, XML file, or some sort of database interface, and
29 the intermediate bulkloader format, which is a dictionary or similar
30 structure representing the external transformation of the data.
31
32 On import, the generate_import_record generator is the only method called.
33
34 On export, the initialize_export method is called once, followed by a call
35 to write_dict for each record, followed by a call to finalize_export.
36
37 The bulkload_state is a BulkloadState object from
38 google.appengine.ext.bulkload.bulkload_config. The interesting properties
39 to a Connector object are the loader_opts and exporter_opts, which are strings
40 passed in from the bulkloader command line.
41 """
42
43 def generate_import_record(self, filename, bulkload_state):
44 """A function which returns an interator over dictionaries.
45
46 This is the only method used on import.
47
48 Args:
49 filename: The --filename argument passed in on the bulkloader command
50 line. This value is opaque to the bulkloader and thus could specify
51 any sort of descriptor for your generator.
52 bulkload_state: Passed in BulkloadConfig.BulkloadState object.
53
54 Returns:
55 An iterator describing an individual record. Typically a dictionary,
56 to be used with dict_to_model. Typically implemented as a generator.
57 """
58 raise NotImplementedError
59
60 def initialize_export(self, filename, bulkload_state):
61 """Initialize the output file.
62
63 Args:
64 filename: The string given as the --filename flag argument.
65 bulkload_state: Passed in BulkloadConfig.BulkloadState object.
66
67 These values are opaque to the bulkloader and thus could specify
68 any sort of descriptor for your exporter.
69 """
70 raise NotImplementedError
71
72 def write_dict(self, dictionary):
73 """Write one record for the specified entity.
74
75 Args:
76 dictionary: A post-transform dictionary.
77 """
78 raise NotImplementedError
79
80 def finalize_export(self):
81 """Performs finalization actions after every record is written."""
82 raise NotImplementedError
83
84
85 def create_from_options(options, name='unknown'):
86 """Factory using an options dictionary.
87
88 This is frequently implemented as the constructor on the connector class,
89 or a static or class method on the connector class.
90
91 Args:
92 options: Parsed dictionary from yaml file, interpretation is up to the
93 implementor of this class.
94 name: Identifier of this transform to be used in error messages.
95
96 Returns:
97 An object which implements the ConnectorInterface interface.
98 """
99 raise NotImplementedError 
diff -uNr google_appengine_1.3.3/google/appengine/ext/bulkload/csv_connector.py google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/csv_connector.py
google_appengine_1.3.3/google/appengine/ext/bulkload/csv_connector.py [1970-01-01 09:00:00.000000000 +0900]
google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/csv_connector.py [2010-05-13 06:32:47.000000000 +0900]
1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 """Bulkloader CSV reading and writing.
19
20 Handle the CSV format specified in a bulkloader.yaml file.
21 """
22
23
24
25
26
27 import codecs
28 import cStringIO
29 import csv
30 import encodings
31 import encodings.ascii
32 import encodings.cp1252
33 import encodings.latin_1
34 import encodings.utf_8
35
36 from google.appengine.ext.bulkload import bulkloader_errors
37 from google.appengine.ext.bulkload import connector_interface
38
39
40 def utf8_recoder(stream, encoding):
41 """Generator that reads an encoded stream and reencodes to UTF-8."""
42 for line in codecs.getreader(encoding)(stream):
43 yield line.encode('utf-8')
44
45
46 class UnicodeDictWriter(object):
47 """Based on UnicodeWriter in http://docs.python.org/library/csv.html."""
48
49 def __init__(self, stream, fieldnames, encoding='utf-8', **kwds):
50 """Initialzer.
51
52 Args:
53 stream: Stream to write to.
54 fieldnames: Fieldnames to pass to the DictWriter.
55 encoding: Desired encoding.
56 kwds: Additional arguments to pass to the DictWriter.
57 """
58 writer = codecs.getwriter(encoding)
59 if (writer is encodings.utf_8.StreamWriter or
60 writer is encodings.ascii.StreamWriter or
61 writer is encodings.latin_1.StreamWriter or
62 writer is encodings.cp1252.StreamWriter):
63 self.no_recoding = True
64 self.encoder = codecs.getencoder(encoding)
65 self.writer = csv.DictWriter(stream, fieldnames, **kwds)
66 else:
67 self.no_recoding = False
68 self.encoder = codecs.getencoder('utf-8')
69 self.queue = cStringIO.StringIO()
70 self.writer = csv.DictWriter(self.queue, fieldnames, **kwds)
71 self.stream = writer(stream)
72
73 def writerow(self, row):
74 """Wrap writerow method."""
75 row_encoded = dict([(k, self.encoder(v)[0]) for (k, v) in row.iteritems()])
76 self.writer.writerow(row_encoded)
77 if self.no_recoding:
78 return
79
80 data = self.queue.getvalue()
81 data = data.decode('utf-8')
82 self.stream.write(data)
83 self.queue.truncate(0)
84
85
86 class CsvConnector(connector_interface.ConnectorInterface):
87 """Read/write a (possibly encoded) CSV file."""
88
89 @classmethod
90 def create_from_options(cls, options, name):
91 """Factory using an options dictionary.
92
93 Args:
94 options: Dictionary of options:
95 columns: 'from_header' or blank.
96 column_list: overrides columns specifically.
97 encoding: encoding of the file. e.g. 'utf-8' (default), 'windows-1252'.
98 skip_import_header_row: True to ignore the header line on import.
99 Defaults False, except must be True if columns=from_header.
100 print_export_header_row: True to print a header line on export.
101 Defaults to False except if columns=from_header.
102 import_options: Other kwargs to pass in, like "dialect".
103 export_options: Other kwargs to pass in, like "dialect".
104 name: The name of this transformer, for use in error messages.
105
106 Returns:
107 CsvConnector object described by the specified options.
108
109 Raises:
110 InvalidConfiguration: If the config is invalid.
111 """
112 column_list = options.get('column_list', None)
113 columns = None
114 if not column_list:
115 columns = options.get('columns', 'from_header')
116 if columns != 'from_header':
117 raise bulkloader_errors.InvalidConfiguration(
118 'CSV columns must be "from_header", or a column_list '
119 'must be specified. (In transformer name %s.)' % name)
120 csv_encoding = options.get('encoding', 'utf-8')
121
122
123 skip_import_header_row = options.get('skip_import_header_row',
124 columns == 'from_header')
125 if columns == 'from_header' and not skip_import_header_row:
126 raise bulkloader_errors.InvalidConfiguration(
127 'When CSV columns are "from_header", the header row must always '
128 'be skipped. (In transformer name %s.)' % name)
129 print_export_header_row = options.get('print_export_header_row',
130 columns == 'from_header')
131 import_options = options.get('import_options', {})
132 export_options = options.get('export_options', {})
133 return cls(columns, column_list, skip_import_header_row,
134 print_export_header_row, csv_encoding, import_options,
135 export_options)
136
137 def __init__(self, columns, column_list, skip_import_header_row,
138 print_export_header_row, csv_encoding=None,
139 import_options=None, export_options=None):
140 """Initializer.
141
142 Args:
143 columns: 'from_header' or blank
144 column_list: overrides columns specifically.
145 skip_import_header_row: True to ignore the header line on import.
146 Defaults False, except must be True if columns=from_header.
147 print_export_header_row: True to print a header line on export.
148 Defaults to False except if columns=from_header.
149 csv_encoding: encoding of the file.
150 import_options: Other kwargs to pass in, like "dialect".
151 export_options: Other kwargs to pass in, like "dialect".
152 """
153
154 self.columns = columns
155 self.from_header = (columns == 'from_header')
156 self.column_list = column_list
157 self.skip_import_header_row = skip_import_header_row
158 self.print_export_header_row = print_export_header_row
159 self.csv_encoding = csv_encoding
160 self.dict_generator = None
161 self.output_stream = None
162 self.csv_writer = None
163 self.bulkload_state = None
164 self.import_options = import_options or {}
165 self.export_options = export_options or {}
166
167 def generate_import_record(self, filename, bulkload_state):
168 """Generator, yields dicts for nodes found as described in the options.
169
170 Args:
171 filename: Filename to read.
172 bulkload_state: Passed bulkload_state.
173
174 Yields:
175 Neutral dict, one per row in the CSV file.
176 """
177 self.bulkload_state = bulkload_state
178 input_stream = open(filename)
179 input_stream = utf8_recoder(input_stream, self.csv_encoding)
180
181 self.dict_generator = csv.DictReader(input_stream, self.column_list,
182 **self.import_options)
183 discard_line = self.skip_import_header_row and not self.from_header
184
185 for input_dict in self.dict_generator:
186 if discard_line:
187 discard_line = False
188 continue
189
190 decoded_dict = {}
191 for key, value in input_dict.iteritems():
192 if not self.column_list:
193 key = unicode(key, 'utf-8')
194 if value:
195 value = unicode(value, 'utf-8')
196 decoded_dict[key] = value
197 yield decoded_dict
198
199 def initialize_export(self, filename, bulkload_state):
200 """Initialize the output file.
201
202 Args:
203 filename: Filename to write.
204 bulkload_state: Passed bulkload_state.
205 """
206 self.bulkload_state = bulkload_state
207 self.output_stream = open(filename, 'wb')
208
209 def __initialize_csv_writer(self, dictionary):
210 """Actual initialization, happens on the first entity being written."""
211 write_header = self.print_export_header_row
212 if self.from_header:
213 export_column_list = tuple(dictionary)
214 else:
215 export_column_list = self.column_list
216
217 self.csv_writer = UnicodeDictWriter(self.output_stream, export_column_list,
218 self.csv_encoding,
219 **self.export_options)
220 if write_header:
221 self.csv_writer.writerow(dict(zip(export_column_list,