Paste #81

google_appengine_1.3.3/README [2010-04-20 03:02:10.000000000 +0900]

google_appengine_1.3.4_prerelease/README [2010-05-13 06:32:44.000000000 +0900]

…
54	54	Address to which this server should bind. (Default
55	55	localhost).
56	56	--port=PORT, -p PORT Port for the server to run on. (Default 8080)
	57	--blobstore_path=PATH Path to use for storing Blobstore file stub data.
57	58	--datastore_path=PATH Path to use for storing Datastore file stub data.
58	59	(Default /tmp/dev_appserver.datastore)
	60	--use_sqlite Use the new, SQLite based datastore stub.
	61	(Default false)
59	62	--history_path=PATH Path to use for storing Datastore history.
60	63	(Default /tmp/dev_appserver.datastore.history)
61	64	--require_indexes Disallows queries that require composite indexes
79	82	--debug_imports Enables debug logging for module imports, showing
80	83	search paths used for finding modules and any
81	84	errors encountered during the import process.
	85	--allow_skipped_files Allow access to files matched by app.yaml's
	86	skipped_files (default False)
82	87	--disable_static_caching Never allow the browser to cache static files.
83	88	(Default enable if expiration set in app.yaml)
	89	--disable_task_running When supplied, tasks will not be automatically
	90	run after submission and must be run manually
	91	in the local admin console.
	92	--task_retry_seconds How long to wait in seconds before retrying a
	93	task after it fails during execution.
	94	(Default '30')
84	95
85	96
86	97

google_appengine_1.3.3/RELEASE_NOTES [2010-04-20 03:02:10.000000000 +0900]

google_appengine_1.3.4_prerelease/RELEASE_NOTES [2010-05-13 06:32:43.000000000 +0900]

3	3
4	4	App Engine Python SDK - Release Notes
5	5
	6	Version 1.3.4
	7	=================================
	8	- New bulkloader configuration syntax and wizard for easier import/export with
	9	the datastore.
	10	- Applications can now be configured to authenticate with OpenID by selecting
	11	the OpenID option when creating your application in the admin console.
	12	http://code.google.com/p/googleappengine/issues/detail?id=248
	13	http://code.google.com/p/googleappengine/issues/detail?id=56
	14	- New API to allow App Engine apps to act as OAuth service providers.
	15	http://code.google.com/p/googleappengine/issues/detail?id=919
	16	- Auto task execution is now enabled in the dev_appserver. To turn this off
	17	use the flag --disable_task_running.
	18	- Fixed an issue using db.put() with constructor initialized id based keys.
	19	http://code.google.com/p/googleappengine/issues/detail?id=3209
	20
6	21	Version 1.3.3
7	22	=================================
8	23	- A new experimental feature allows you to set dev_appserver datastore file

google_appengine_1.3.3/VERSION [2010-04-20 03:02:10.000000000 +0900]

google_appengine_1.3.4_prerelease/VERSION [2010-05-13 07:29:11.000000000 +0900]

1		release: "1.3.3"
2		timestamp: 1270494723
	1	release: "prerelease-1.3.4"
	2	timestamp: 1272392128
3	3	api_versions: ['1']

google_appengine_1.3.3/google/appengine/api/datastore_types.py [2010-04-20 03:02:32.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/datastore_types.py [2010-05-13 06:32:53.000000000 +0900]

…
118	118	negative_ok=False):
119	119	"""Raises an exception if value is not a valid integer.
120	120
121		An integer is valid if it's not negative or empty and is an integer.
122		The exception type can be specified with the exception argument;
123		~~it defaults to BadValueError.~~
	121	An integer is valid if it's not negative or empty and is an integer
	122	(either int or long). The exception type raised can be specified
	123	with the exception argument; it defaults to BadValueError.
124	124
125	125	Args:
126	126	value: the value to validate.
132	132	"""
133	133	if value is None and empty_ok:
134	134	return
135		if not isinstance(value, ~~int~~):
	135	if not isinstance(value, (int, long)):
136	136	raise exception('%s should be an integer; received %s (a %s).' %
137	137	(name, value, typename(value)))
138	138	if not value and not zero_ok:
…
1374	1374	pbvalue.mutable_uservalue().set_obfuscated_gaiaid(
1375	1375	value.user_id().encode('utf-8'))
1376	1376
	1377	if value.federated_identity() is not None:
	1378	pbvalue.mutable_uservalue().set_federated_identity(
	1379	value.federated_identity().encode('utf-8'))
	1380
	1381	if value.federated_provider() is not None:
	1382	pbvalue.mutable_uservalue().set_federated_provider(
	1383	value.federated_provider().encode('utf-8'))
	1384
1377	1385
1378	1386	def PackKey(name, value, pbvalue):
1379	1387	"""Packs a reference property into a entity_pb.PropertyValue.
…
1577	1585	auth_domain = unicode(pbval.uservalue().auth_domain().decode('utf-8'))
1578	1586	obfuscated_gaiaid = pbval.uservalue().obfuscated_gaiaid().decode('utf-8')
1579	1587	obfuscated_gaiaid = unicode(obfuscated_gaiaid)
	1588
	1589	federated_identity = None
	1590	if pbval.uservalue().has_federated_identity():
	1591	federated_identity = unicode(
	1592	pbval.uservalue().federated_identity().decode('utf-8'))
	1593
1580	1594	value = users.User(email=email,
1581	1595	_auth_domain=auth_domain,
1582		_user_id=obfuscated_gaiaid)
	1596	_user_id=obfuscated_gaiaid,
	1597	federated_identity=federated_identity)
1583	1598	else:
1584	1599	value = None
1585	1600
…
1703	1718	elif type_ == type(None):
1704	1719	return None
1705	1720	return type_(value_string)
1706

google_appengine_1.3.3/google/appengine/api/labs/taskqueue/taskqueue_stub.py [2010-04-20 03:02:33.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/labs/taskqueue/taskqueue_stub.py [2010-05-13 06:32:56.000000000 +0900]

…
17	17
18	18	"""Stub version of the Task Queue API.
19	19
20		This stub only stores tasks; it doesn't actually run them. It also validates
21		the tasks by checking their queue name against the queue.yaml.
	20	This stub stores tasks and runs them via dev_appserver's AddEvent capability.
	21	It also validates the tasks by checking their queue name against the queue.yaml.
22	22
23	23	As well as implementing Task Queue API functions, the stub exposes various other
24	24	functions that are used by the dev_appserver's admin console to display the
28	28
29	29
30	30
	31	import StringIO
31	32	import base64
32	33	import bisect
33	34	import datetime
…
300	301	class TaskQueueServiceStub(apiproxy_stub.APIProxyStub):
301	302	"""Python only task queue service stub.
302	303
303		This stub does not attempt to automatically execute tasks. Instead, it
304		stores them for display on a console. The user may manually execute the
305		~~tasks from the console.~~
	304	This stub executes tasks when enabled by using the dev_appserver's AddEvent
	305	capability. When task running is disabled this stub will store tasks for
	306	display on a console, where the user may manually execute the tasks.
306	307	"""
307	308
308	309	queue_yaml_parser = _ParseQueueYaml
309	310
310		def __init__(self, ~~service_name='taskqueue', root_path=None):~~
	311	def __init__(self,
	312	service_name='taskqueue',
	313	root_path=None,
	314	auto_task_running=False,
	315	task_retry_seconds=30):
311	316	"""Constructor.
312	317
313	318	Args:
…
315	320	root_path: Root path to the directory of the application which may contain
316	321	a queue.yaml file. If None, then it's assumed no queue.yaml file is
317	322	available.
	323	auto_task_running: When True, the dev_appserver should automatically
	324	run tasks after they are enqueued.
	325	task_retry_seconds: How long to wait between task executions after a
	326	task fails.
318	327	"""
319	328	super(TaskQueueServiceStub, self).__init__(service_name)
320	329	self._taskqueues = {}
321	330	self._next_task_id = 1
322	331	self._root_path = root_path
323	332
	333	self._add_event = None
	334	self._auto_task_running = auto_task_running
	335	self._task_retry_seconds = task_retry_seconds
	336
324	337	self._app_queues = {}
325	338
326	339	def _ChooseTaskName(self):
…
480	493	taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS)
481	494	else:
482	495	existing_tasks.append(add_request)
	496
	497	if self._add_event and self._auto_task_running:
	498	self._add_event(
	499	add_request.eta_usec() / 1000000.0,
	500	lambda: self._RunTask(
	501	add_request.queue_name(), add_request.task_name()))
	502
483	503	existing_tasks.sort(_CompareTasksByEta)
484	504
485	505	def _IsValidQueue(self, queue_name):
…
503	523	return True
504	524	return False
505	525
	526	def _RunTask(self, queue_name, task_name):
	527	"""Returns a fake request for running a task in the dev_appserver.
	528
	529	Args:
	530	queue_name: The queue the task is in.
	531	task_name: The name of the task to run.
	532
	533	Returns:
	534	None if this task no longer exists or tuple (connection, addrinfo) of
	535	a fake connection and address information used to run this task. The
	536	task will be deleted after it runs or re-enqueued in the future on
	537	failure.
	538	"""
	539	task_list = self.GetTasks(queue_name)
	540	for task in task_list:
	541	if task['name'] == task_name:
	542	break
	543	else:
	544	return None
	545
	546	class FakeConnection(object):
	547	def __init__(self, input_buffer):
	548	self.rfile = StringIO.StringIO(input_buffer)
	549	self.wfile = StringIO.StringIO()
	550	self.wfile_close = self.wfile.close
	551	self.wfile.close = self.connection_done
	552
	553	def connection_done(myself):
	554	result = myself.wfile.getvalue()
	555	myself.wfile_close()
	556	first_line, rest = result.split('\n', 1)
	557	version, code, rest = first_line.split(' ', 2)
	558	if 200 <= int(code) <= 299:
	559	self.DeleteTask(queue_name, task_name)
	560	return
	561
	562	logging.warning('Task named "%s" on queue "%s" failed with code %s; '
	563	'will retry in %d seconds',
	564	task_name, queue_name, code, self._task_retry_seconds)
	565	self._add_event(
	566	time.time() + self._task_retry_seconds,
	567	lambda: self._RunTask(queue_name, task_name))
	568
	569	def close(self):
	570	pass
	571
	572	def makefile(self, mode, buffsize):
	573	if mode.startswith('w'):
	574	return self.wfile
	575	else:
	576	return self.rfile
	577
	578	payload = StringIO.StringIO()
	579	payload.write('%s %s HTTP/1.1\r\n' % (task['method'], task['url']))
	580	for key, value in task['headers']:
	581	payload.write('%s: %s\r\n' % (key, value))
	582	payload.write('\r\n')
	583	payload.write(task['body'])
	584
	585	return FakeConnection(payload.getvalue()), ('0.1.0.2', 80)
	586
506	587	def GetQueues(self):
507	588	"""Gets all the applications's queues.
508	589

google_appengine_1.3.3/google/appengine/api/oauth/__init__.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/oauth/__init__.py [2010-05-13 06:32:55.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""OAuth API module."""
	19
	20	from oauth_api import *
	21
	22
	23	__all__ = ['Error',
	24	'OAuthRequestError',
	25	'NotAllowedError',
	26	'InvalidOAuthParametersError',
	27	'InvalidOAuthTokenError',
	28	'OAuthServiceFailureError',
	29	'get_current_user',
	30	'is_current_user_admin',
	31	'get_oauth_consumer_key',
	32	]

google_appengine_1.3.3/google/appengine/api/oauth/oauth_api.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/oauth/oauth_api.py [2010-05-13 06:32:55.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""OAuth API.
	19
	20	A service that enables App Engine apps to validate OAuth requests.
	21
	22	Classes defined here:
	23	Error: base exception type
	24	NotAllowedError: OAuthService exception
	25	OAuthRequestError: OAuthService exception
	26	InvalidOAuthParametersError: OAuthService exception
	27	InvalidOAuthTokenError: OAuthService exception
	28	OAuthServiceFailureError: OAuthService exception
	29	"""
	30
	31
	32
	33
	34
	35
	36
	37	import os
	38
	39	from google.appengine.api import apiproxy_stub_map
	40	from google.appengine.api import user_service_pb
	41	from google.appengine.api import users
	42	from google.appengine.runtime import apiproxy_errors
	43
	44
	45	class Error(Exception):
	46	"""Base error class for this module."""
	47
	48
	49	class OAuthRequestError(Error):
	50	"""Base error type for invalid OAuth requests."""
	51
	52
	53	class NotAllowedError(OAuthRequestError):
	54	"""Raised if the requested URL does not permit OAuth authentication."""
	55
	56
	57	class InvalidOAuthParametersError(OAuthRequestError):
	58	"""Raised if the request was a malformed OAuth request.
	59
	60	For example, the request may have omitted a required parameter, contained
	61	an invalid signature, or was made by an unknown consumer.
	62	"""
	63
	64
	65	class InvalidOAuthTokenError(OAuthRequestError):
	66	"""Raised if the request contained an invalid token.
	67
	68	For example, the token may have been revoked by the user.
	69	"""
	70
	71
	72	class OAuthServiceFailureError(Error):
	73	"""Raised if there was a problem communicating with the OAuth service."""
	74
	75
	76	def get_current_user():
	77	"""Returns the User on whose behalf the request was made.
	78
	79	Returns:
	80	User
	81
	82	Raises:
	83	OAuthRequestError: The request was not a valid OAuth request.
	84	OAuthServiceFailureError: An unknown error occurred.
	85	"""
	86	_maybe_call_get_oauth_user()
	87	return _get_user_from_environ()
	88
	89
	90	def is_current_user_admin():
	91	"""Returns true if the User on whose behalf the request was made is an admin.
	92
	93	Returns:
	94	boolean
	95
	96	Raises:
	97	OAuthRequestError: The request was not a valid OAuth request.
	98	OAuthServiceFailureError: An unknown error occurred.
	99	"""
	100	_maybe_call_get_oauth_user()
	101	return os.environ.get('OAUTH_IS_ADMIN', '0') == '1'
	102
	103
	104	def get_oauth_consumer_key():
	105	"""Returns the value of the 'oauth_consumer_key' parameter from the request.
	106
	107	Returns:
	108	string: The value of the 'oauth_consumer_key' parameter from the request,
	109	an identifier for the consumer that signed the request.
	110
	111	Raises:
	112	OAuthRequestError: The request was not a valid OAuth request.
	113	OAuthServiceFailureError: An unknown error occurred.
	114	"""
	115	req = user_service_pb.CheckOAuthSignatureRequest()
	116	resp = user_service_pb.CheckOAuthSignatureResponse()
	117	try:
	118	apiproxy_stub_map.MakeSyncCall('user', 'CheckOAuthSignature', req, resp)
	119	except apiproxy_errors.ApplicationError, e:
	120	if (e.application_error ==
	121	user_service_pb.UserServiceError.OAUTH_INVALID_REQUEST):
	122	raise InvalidOAuthParametersError
	123	elif (e.application_error ==
	124	user_service_pb.UserServiceError.OAUTH_ERROR):
	125	raise OAuthServiceFailureError
	126	else:
	127	raise OAuthServiceFailureError
	128	return resp.oauth_consumer_key()
	129
	130
	131	def _maybe_call_get_oauth_user():
	132	"""Makes an GetOAuthUser RPC and stores the results in os.environ.
	133
	134	This method will only make the RPC if 'OAUTH_ERROR_CODE' has not already
	135	been set.
	136	"""
	137	if 'OAUTH_ERROR_CODE' not in os.environ:
	138	req = user_service_pb.GetOAuthUserRequest()
	139	resp = user_service_pb.GetOAuthUserResponse()
	140	try:
	141	apiproxy_stub_map.MakeSyncCall('user', 'GetOAuthUser', req, resp)
	142	os.environ['OAUTH_EMAIL'] = resp.email()
	143	os.environ['OAUTH_AUTH_DOMAIN'] = resp.auth_domain()
	144	os.environ['OAUTH_USER_ID'] = resp.user_id()
	145	if resp.is_admin():
	146	os.environ['OAUTH_IS_ADMIN'] = '1'
	147	else:
	148	os.environ['OAUTH_IS_ADMIN'] = '0'
	149	os.environ['OAUTH_ERROR_CODE'] = ''
	150	except apiproxy_errors.ApplicationError, e:
	151	os.environ['OAUTH_ERROR_CODE'] = str(e.application_error)
	152	_maybe_raise_exception()
	153
	154
	155	def _maybe_raise_exception():
	156	"""Raises an error if one has been stored in os.environ.
	157
	158	This method requires that 'OAUTH_ERROR_CODE' has already been set (an empty
	159	string indicates that there is no actual error).
	160	"""
	161	assert 'OAUTH_ERROR_CODE' in os.environ
	162	error = os.environ['OAUTH_ERROR_CODE']
	163	if error:
	164	if error == str(user_service_pb.UserServiceError.NOT_ALLOWED):
	165	raise NotAllowedError
	166	elif error == str(user_service_pb.UserServiceError.OAUTH_INVALID_REQUEST):
	167	raise InvalidOAuthParametersError
	168	elif error == str(user_service_pb.UserServiceError.OAUTH_INVALID_TOKEN):
	169	raise InvalidOAuthTokenError
	170	elif error == str(user_service_pb.UserServiceError.OAUTH_ERROR):
	171	raise OAuthServiceFailureError
	172	else:
	173	raise OAuthServiceFailureError
	174
	175
	176	def _get_user_from_environ():
	177	"""Returns a User based on values stored in os.environ.
	178
	179	This method requires that 'OAUTH_EMAIL', 'OAUTH_AUTH_DOMAIN', and
	180	'OAUTH_USER_ID' have already been set.
	181
	182	Returns:
	183	User
	184	"""
	185	assert 'OAUTH_EMAIL' in os.environ
	186	assert 'OAUTH_AUTH_DOMAIN' in os.environ
	187	assert 'OAUTH_USER_ID' in os.environ
	188	return users.User(email=os.environ['OAUTH_EMAIL'],
	189	_auth_domain=os.environ['OAUTH_AUTH_DOMAIN'],
	190	_user_id=os.environ['OAUTH_USER_ID'])

google_appengine_1.3.3/google/appengine/api/user_service_pb.py [2010-04-20 03:02:16.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/user_service_pb.py [2010-05-13 06:32:35.000000000 +0900]

…
102	102	destination_url_ = ""
103	103	has_auth_domain_ = 0
104	104	auth_domain_ = ""
	105	has_federated_identity_ = 0
	106	federated_identity_ = "google.com"
105	107
106	108	def __init__(self, contents=None):
107	109	if contents is not None: self.MergeFromString(contents)
132	134
133	135	def has_auth_domain(self): return self.has_auth_domain_
134	136
	137	def federated_identity(self): return self.federated_identity_
	138
	139	def set_federated_identity(self, x):
	140	self.has_federated_identity_ = 1
	141	self.federated_identity_ = x
	142
	143	def clear_federated_identity(self):
	144	if self.has_federated_identity_:
	145	self.has_federated_identity_ = 0
	146	self.federated_identity_ = "google.com"
	147
	148	def has_federated_identity(self): return self.has_federated_identity_
	149
135	150
136	151	def MergeFrom(self, x):
137	152	assert x is not self
138	153	if (x.has_destination_url()): self.set_destination_url(x.destination_url())
139	154	if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain())
	155	if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
140	156
141	157	def Equals(self, x):
142	158	if x is self: return 1
…
144	160	if self.has_destination_url_ and self.destination_url_ != x.destination_url_: return 0
145	161	if self.has_auth_domain_ != x.has_auth_domain_: return 0
146	162	if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0
	163	if self.has_federated_identity_ != x.has_federated_identity_: return 0
	164	if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
147	165	return 1
148	166
149	167	def IsInitialized(self, debug_strs=None):
…
158	176	n = 0
159	177	n += self.lengthString(len(self.destination_url_))
160	178	if (self.has_auth_domain_): n += 1 + self.lengthString(len(self.auth_domain_))
	179	if (self.has_federated_identity_): n += 1 + self.lengthString(len(self.federated_identity_))
161	180	return n + 1
162	181
163	182	def Clear(self):
164	183	self.clear_destination_url()
165	184	self.clear_auth_domain()
	185	self.clear_federated_identity()
166	186
167	187	def OutputUnchecked(self, out):
168	188	out.putVarInt32(10)
…
170	190	if (self.has_auth_domain_):
171	191	out.putVarInt32(18)
172	192	out.putPrefixedString(self.auth_domain_)
	193	if (self.has_federated_identity_):
	194	out.putVarInt32(26)
	195	out.putPrefixedString(self.federated_identity_)
173	196
174	197	def TryMerge(self, d):
175	198	while d.avail() > 0:
…
180	203	if tt == 18:
181	204	self.set_auth_domain(d.getPrefixedString())
182	205	continue
	206	if tt == 26:
	207	self.set_federated_identity(d.getPrefixedString())
	208	continue
183	209	if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
184	210	d.skipData(tt)
185	211
…
188	214	res=""
189	215	if self.has_destination_url_: res+=prefix+("destination_url: %s\n" % self.DebugFormatString(self.destination_url_))
190	216	if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_))
	217	if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
191	218	return res
192	219
193	220
…
196	223
197	224	kdestination_url = 1
198	225	kauth_domain = 2
	226	kfederated_identity = 3
199	227
200	228	_TEXT = _BuildTagLookupTable({
201	229	0: "ErrorCode",
202	230	1: "destination_url",
203	231	2: "auth_domain",
204		~~}, 2)~~
	232	3: "federated_identity",
	233	}, 3)
205	234
206	235	_TYPES = _BuildTagLookupTable({
207	236	0: ProtocolBuffer.Encoder.NUMERIC,
208	237	1: ProtocolBuffer.Encoder.STRING,
209	238	2: ProtocolBuffer.Encoder.STRING,
210		~~}, 2, ProtocolBuffer.Encoder.MAX_TYPE)~~
	239	3: ProtocolBuffer.Encoder.STRING,
	240	}, 3, ProtocolBuffer.Encoder.MAX_TYPE)
211	241
212	242	_STYLE = """"""
213	243	_STYLE_CONTENT_TYPE = """"""
…
556	586	auth_domain_ = ""
557	587	has_user_organization_ = 0
558	588	user_organization_ = ""
	589	has_is_admin_ = 0
	590	is_admin_ = 0
559	591
560	592	def __init__(self, contents=None):
561	593	if contents is not None: self.MergeFromString(contents)
…
612	644
613	645	def has_user_organization(self): return self.has_user_organization_
614	646
	647	def is_admin(self): return self.is_admin_
	648
	649	def set_is_admin(self, x):
	650	self.has_is_admin_ = 1
	651	self.is_admin_ = x
	652
	653	def clear_is_admin(self):
	654	if self.has_is_admin_:
	655	self.has_is_admin_ = 0
	656	self.is_admin_ = 0
	657
	658	def has_is_admin(self): return self.has_is_admin_
	659
615	660
616	661	def MergeFrom(self, x):
617	662	assert x is not self
…
619	664	if (x.has_user_id()): self.set_user_id(x.user_id())
620	665	if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain())
621	666	if (x.has_user_organization()): self.set_user_organization(x.user_organization())
	667	if (x.has_is_admin()): self.set_is_admin(x.is_admin())
622	668
623	669	def Equals(self, x):
624	670	if x is self: return 1
…
630	676	if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0
631	677	if self.has_user_organization_ != x.has_user_organization_: return 0
632	678	if self.has_user_organization_ and self.user_organization_ != x.user_organization_: return 0
	679	if self.has_is_admin_ != x.has_is_admin_: return 0
	680	if self.has_is_admin_ and self.is_admin_ != x.is_admin_: return 0
633	681	return 1
634	682
635	683	def IsInitialized(self, debug_strs=None):
…
654	702	n += self.lengthString(len(self.user_id_))
655	703	n += self.lengthString(len(self.auth_domain_))
656	704	if (self.has_user_organization_): n += 1 + self.lengthString(len(self.user_organization_))
	705	if (self.has_is_admin_): n += 2
657	706	return n + 3
658	707
659	708	def Clear(self):
…
661	710	self.clear_user_id()
662	711	self.clear_auth_domain()
663	712	self.clear_user_organization()
	713	self.clear_is_admin()
664	714
665	715	def OutputUnchecked(self, out):
666	716	out.putVarInt32(10)
…
672	722	if (self.has_user_organization_):
673	723	out.putVarInt32(34)
674	724	out.putPrefixedString(self.user_organization_)
	725	if (self.has_is_admin_):
	726	out.putVarInt32(40)
	727	out.putBoolean(self.is_admin_)
675	728
676	729	def TryMerge(self, d):
677	730	while d.avail() > 0:
…
688	741	if tt == 34:
689	742	self.set_user_organization(d.getPrefixedString())
690	743	continue
	744	if tt == 40:
	745	self.set_is_admin(d.getBoolean())
	746	continue
691	747	if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
692	748	d.skipData(tt)
693	749
…
698	754	if self.has_user_id_: res+=prefix+("user_id: %s\n" % self.DebugFormatString(self.user_id_))
699	755	if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_))
700	756	if self.has_user_organization_: res+=prefix+("user_organization: %s\n" % self.DebugFormatString(self.user_organization_))
	757	if self.has_is_admin_: res+=prefix+("is_admin: %s\n" % self.DebugFormatBool(self.is_admin_))
701	758	return res
702	759
703	760
…
708	765	kuser_id = 2
709	766	kauth_domain = 3
710	767	kuser_organization = 4
	768	kis_admin = 5
711	769
712	770	_TEXT = _BuildTagLookupTable({
713	771	0: "ErrorCode",
…
715	773	2: "user_id",
716	774	3: "auth_domain",
717	775	4: "user_organization",
718		~~}, 4)~~
	776	5: "is_admin",
	777	}, 5)
719	778
720	779	_TYPES = _BuildTagLookupTable({
721	780	0: ProtocolBuffer.Encoder.NUMERIC,
…
723	782	2: ProtocolBuffer.Encoder.STRING,
724	783	3: ProtocolBuffer.Encoder.STRING,
725	784	4: ProtocolBuffer.Encoder.STRING,
726		~~}, 4, ProtocolBuffer.Encoder.MAX_TYPE)~~
	785	5: ProtocolBuffer.Encoder.NUMERIC,
	786	}, 5, ProtocolBuffer.Encoder.MAX_TYPE)
727	787
728	788	_STYLE = """"""
729	789	_STYLE_CONTENT_TYPE = """"""

google_appengine_1.3.3/google/appengine/api/user_service_stub.py [2010-04-20 03:02:36.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/user_service_stub.py [2010-05-13 06:32:55.000000000 +0900]

…
28	28	_DEFAULT_LOGIN_URL = 'https://www.google.com/accounts/Login?continue=%s'
29	29	_DEFAULT_LOGOUT_URL = 'https://www.google.com/accounts/Logout?continue=%s'
30	30
	31	_OAUTH_CONSUMER_KEY = 'example.com'
	32	_OAUTH_EMAIL = 'example@example.com'
	33	_OAUTH_USER_ID = '0'
	34	_OAUTH_AUTH_DOMAIN = 'gmail.com'
	35
31	36
32	37	class UserServiceStub(apiproxy_stub.APIProxyStub):
33	38	"""Trivial implementation of the UserService."""
61	66	"""Trivial implementation of UserService.CreateLoginURL().
62	67
63	68	Args:
64		request: the URL to redirect to after login; a base.StringProto
65		response: the login URL; a base.StringProto
	69	request: a CreateLoginURLRequest
	70	response: a CreateLoginURLResponse
66	71	"""
67	72	self.__num_requests += 1
68	73	response.set_login_url(
…
73	78	"""Trivial implementation of UserService.CreateLogoutURL().
74	79
75	80	Args:
76		request: the URL to redirect to after logout; a base.StringProto
77		response: the logout URL; a base.StringProto
	81	request: a CreateLogoutURLRequest
	82	response: a CreateLogoutURLResponse
78	83	"""
79	84	self.__num_requests += 1
80	85	response.set_logout_url(
81	86	self._logout_url %
82	87	urllib.quote(self._AddHostToContinueURL(request.destination_url())))
83	88
	89	def _Dynamic_GetOAuthUser(self, unused_request, response):
	90	"""Trivial implementation of UserService.GetOAuthUser().
	91
	92	Args:
	93	unused_request: a GetOAuthUserRequest
	94	response: a GetOAuthUserResponse
	95	"""
	96	self.__num_requests += 1
	97	response.set_email(_OAUTH_EMAIL)
	98	response.set_user_id(_OAUTH_USER_ID)
	99	response.set_auth_domain(_OAUTH_AUTH_DOMAIN)
	100
	101	def _Dynamic_CheckOAuthSignature(self, unused_request, response):
	102	"""Trivial implementation of UserService.CheckOAuthSignature().
	103
	104	Args:
	105	unused_request: a CheckOAuthSignatureRequest
	106	response: a CheckOAuthSignatureResponse
	107	"""
	108	self.__num_requests += 1
	109	response.set_oauth_consumer_key(_OAUTH_CONSUMER_KEY)
	110
84	111	def _AddHostToContinueURL(self, continue_url):
85	112	"""Adds the request host to the continue url if no host is specified.
86	113

google_appengine_1.3.3/google/appengine/api/users.py [2010-04-20 03:02:36.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/api/users.py [2010-05-13 06:32:53.000000000 +0900]

…
18	18	"""Python datastore class User to be used as a datastore data type.
19	19
20	20	Classes defined here:
21		User: object representing a user.
	21	User: object representing a user. A user could be a Google Accounts user
	22	or a federated user.
22	23	Error: base exception type
23	24	UserNotFoundError: UserService exception
24	25	RedirectTooLongError: UserService exception
29	30
30	31
31	32
32
33	33	import os
34	34	from google.appengine.api import apiproxy_stub_map
35	35	from google.appengine.api import user_service_pb
…
63	63	A nickname is a human-readable string which uniquely identifies a Google
64	64	user, akin to a username. It will be an email address for some users, but
65	65	not all.
	66
	67	A user could be a Google Accounts user or a federated login user.
	68
	69	federated_identity and federated_provider are only avaliable for
	70	federated users.
66	71	"""
67	72
68	73
69	74	__user_id = None
70	75
71		def __init__(self, email=None, _auth_domain=None, ~~_user_id=None):~~
	76	def __init__(self, email=None, _auth_domain=None,
	77	_user_id=None, federated_identity=None, federated_provider=None):
72	78	"""Constructor.
73	79
74	80	Args:
75	81	email: An optional string of the user's email address. It defaults to
76	82	the current user's email address.
	83	federated_identity: federated identity of user. It defaults to the current
	84	user's federated identity.
	85	federated_provider: federated provider url of user.
77	86
78	87	Raises:
79		UserNotFoundError: Raised if the user is not logged in and the email
80		argument is empty.
	88	UserNotFoundError: Raised if the user is not logged in and both email
	89	and federated identity are empty.
81	90	"""
82	91	if _auth_domain is None:
83	92	_auth_domain = os.environ.get('AUTH_DOMAIN')
84		else:
85		assert email is not None
86
87	93	assert _auth_domain
88	94
89		if email is None:
90		assert 'USER_EMAIL' in os.environ
91		email = os.environ['USER_EMAIL']
92		if _user_id is None and 'USER_ID' in os.environ:
93		~~_user_id = os.environ['USER_ID']~~
	95	if email is None and federated_identity is None:
	96	email = os.environ.get('USER_EMAIL', email)
	97	_user_id = os.environ.get('USER_ID', _user_id)
	98	federated_identity = os.environ.get('FEDERATED_IDENTITY',
	99	federated_identity)
	100	federated_provider = os.environ.get('FEDERATED_PROVIDER',
	101	federated_provider)
94	102
95		if not email:
	103	if not email and not federated_identity:
96	104	raise UserNotFoundError
97	105
98	106	self.__email = email
	107	self.__federated_identity = federated_identity
	108	self.__federated_provider = federated_provider
99	109	self.__auth_domain = _auth_domain
100	110	self.__user_id = _user_id or None
101	111
…
104	114
105	115	The nickname will be a unique, human readable identifier for this user
106	116	with respect to this application. It will be an email address for some
107		users, ~~but not all.~~
	117	users, part of the email address for some users, and the federated identity
	118	for federated users who have not asserted an email address.
108	119	"""
109	120	if (self.__email and self.__auth_domain and
110	121	self.__email.endswith('@' + self.__auth_domain)):
111	122	suffix_len = len(self.__auth_domain) + 1
112	123	return self.__email[:-suffix_len]
	124	elif self.__federated_identity:
	125	return self.__federated_identity
113	126	else:
114	127	return self.__email
115	128
…
128	141	"""Return this user's auth domain."""
129	142	return self.__auth_domain
130	143
	144	def federated_identity(self):
	145	"""Return this user's federated identity, None if not a federated user."""
	146	return self.__federated_identity
	147
	148	def federated_provider(self):
	149	"""Return this user's federated provider, None if not a federated user."""
	150	return self.__federated_provider
	151
131	152	def __unicode__(self):
132	153	return unicode(self.nickname())
133	154
…
135	156	return str(self.nickname())
136	157
137	158	def __repr__(self):
	159	values = []
	160	if self.__email:
	161	values.append("email='%s'" % self.__email)
	162	if self.__federated_identity:
	163	values.append("federated_identity='%s'" % self.__federated_identity)
138	164	if self.__user_id:
139		return "users.User(email='%s',_user_id='%s')" % (self.email(),
140		self.user_id())
141		else:
142		return "users.User(email='%s')" % self.email()
	165	values.append("_user_id='%s'" % self.__user_id)
	166	return 'users.User(%s)' % ','.join(values)
143	167
144	168	def __hash__(self):
145		~~return hash((self.__email, self.__auth_domain))~~
	169	if self.__federated_identity:
	170	return hash((self.__federated_identity, self.__auth_domain))
	171	else:
	172	return hash((self.__email, self.__auth_domain))
146	173
147	174	def __cmp__(self, other):
148	175	if not isinstance(other, User):
149	176	return NotImplemented
150		return cmp((self.__email, self.__auth_domain),
151		(other.__email, other.__auth_domain))
	177	if self.__federated_identity:
	178	return cmp((self.__federated_identity, self.__auth_domain),
	179	(other.__federated_identity, other.__auth_domain))
	180	else:
	181	return cmp((self.__email, self.__auth_domain),
	182	(other.__email, other.__auth_domain))
152	183
153	184
154		def create_login_url(dest_url, _auth_domain=None):
155		"""Computes the login URL for this request and specified destination URL.
	185	def create_login_url(dest_url=None, _auth_domain=None,
	186	federated_identity=None):
	187	"""Computes the login URL for redirection.
156	188
157	189	Args:
158	190	dest_url: String that is the desired final destination URL for the user
159	191	once login is complete. If 'dest_url' does not have a host
160	192	specified, we will use the host from the current request.
	193	federated_identity: federated_identity is used to trigger OpenId Login
	194	flow, an empty value will trigger Google OpenID Login
	195	by default.
161	196
162	197	Returns:
163		~~string~~
	198	Login URL as a string. If federated_identity is set, this will be
	199	a federated login using the specified identity. If not, this
	200	will use Google Accounts.
164	201	"""
165	202	req = user_service_pb.CreateLoginURLRequest()
166	203	resp = user_service_pb.CreateLoginURLResponse()
167	204	req.set_destination_url(dest_url)
168	205	if _auth_domain:
169	206	req.set_auth_domain(_auth_domain)
	207	if federated_identity:
	208	req.set_federated_identity(federated_identity)
170	209
171	210	try:
172	211	apiproxy_stub_map.MakeSyncCall('user', 'CreateLoginURL', req, resp)
…
185	224
186	225
187	226	def create_logout_url(dest_url, _auth_domain=None):
188		"""Computes the logout URL for this request and specified destination URL.
	227	"""Computes the logout URL for this request and specified destination URL,
	228	for both federated login App and Google Accounts App.
189	229
190	230	Args:
191	231	dest_url: String that is the desired final destination URL for the user
…
193	233	specified, we will use the host from the current request.
194	234
195	235	Returns:
196		string
	236	Logout URL as a string
197	237	"""
198	238	req = user_service_pb.CreateLogoutURLRequest()
199	239	resp = user_service_pb.CreateLogoutURLResponse()

google_appengine_1.3.3/google/appengine/datastore/entity_pb.py [2010-04-20 03:02:28.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/datastore/entity_pb.py [2010-05-13 06:32:30.000000000 +0900]

…
249	249	gaiaid_ = 0
250	250	has_obfuscated_gaiaid_ = 0
251	251	obfuscated_gaiaid_ = ""
	252	has_federated_identity_ = 0
	253	federated_identity_ = ""
	254	has_federated_provider_ = 0
	255	federated_provider_ = ""
252	256
253	257	def __init__(self, contents=None):
254	258	if contents is not None: self.MergeFromString(contents)
318	322
319	323	def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_
320	324
	325	def federated_identity(self): return self.federated_identity_
	326
	327	def set_federated_identity(self, x):
	328	self.has_federated_identity_ = 1
	329	self.federated_identity_ = x
	330
	331	def clear_federated_identity(self):
	332	if self.has_federated_identity_:
	333	self.has_federated_identity_ = 0
	334	self.federated_identity_ = ""
	335
	336	def has_federated_identity(self): return self.has_federated_identity_
	337
	338	def federated_provider(self): return self.federated_provider_
	339
	340	def set_federated_provider(self, x):
	341	self.has_federated_provider_ = 1
	342	self.federated_provider_ = x
	343
	344	def clear_federated_provider(self):
	345	if self.has_federated_provider_:
	346	self.has_federated_provider_ = 0
	347	self.federated_provider_ = ""
	348
	349	def has_federated_provider(self): return self.has_federated_provider_
	350
321	351
322	352	def MergeFrom(self, x):
323	353	assert x is not self
…
326	356	if (x.has_nickname()): self.set_nickname(x.nickname())
327	357	if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid())
328	358	if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid())
	359	if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
	360	if (x.has_federated_provider()): self.set_federated_provider(x.federated_provider())
329	361
330	362	def Equals(self, x):
331	363	if x is self: return 1
…
339	371	if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0
340	372	if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0
341	373	if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0
	374	if self.has_federated_identity_ != x.has_federated_identity_: return 0
	375	if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
	376	if self.has_federated_provider_ != x.has_federated_provider_: return 0
	377	if self.has_federated_provider_ and self.federated_provider_ != x.federated_provider_: return 0
342	378	return 1
343	379
344	380	def IsInitialized(self, debug_strs=None):
…
364	400	if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_))
365	401	n += self.lengthVarInt64(self.gaiaid_)
366	402	if (self.has_obfuscated_gaiaid_): n += 2 + self.lengthString(len(self.obfuscated_gaiaid_))
	403	if (self.has_federated_identity_): n += 2 + self.lengthString(len(self.federated_identity_))
	404	if (self.has_federated_provider_): n += 2 + self.lengthString(len(self.federated_provider_))
367	405	return n + 4
368	406
369	407	def Clear(self):
…
372	410	self.clear_nickname()
373	411	self.clear_gaiaid()
374	412	self.clear_obfuscated_gaiaid()
	413	self.clear_federated_identity()
	414	self.clear_federated_provider()
375	415
376	416	def OutputUnchecked(self, out):
377	417	out.putVarInt32(74)
…
386	426	if (self.has_obfuscated_gaiaid_):
387	427	out.putVarInt32(154)
388	428	out.putPrefixedString(self.obfuscated_gaiaid_)
	429	if (self.has_federated_identity_):
	430	out.putVarInt32(170)
	431	out.putPrefixedString(self.federated_identity_)
	432	if (self.has_federated_provider_):
	433	out.putVarInt32(178)
	434	out.putPrefixedString(self.federated_provider_)
389	435
390	436	def TryMerge(self, d):
391	437	while 1:
…
406	452	if tt == 154:
407	453	self.set_obfuscated_gaiaid(d.getPrefixedString())
408	454	continue
	455	if tt == 170:
	456	self.set_federated_identity(d.getPrefixedString())
	457	continue
	458	if tt == 178:
	459	self.set_federated_provider(d.getPrefixedString())
	460	continue
409	461	if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
410	462	d.skipData(tt)
411	463
…
417	469	if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_))
418	470	if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_))
419	471	if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_))
	472	if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
	473	if self.has_federated_provider_: res+=prefix+("federated_provider: %s\n" % self.DebugFormatString(self.federated_provider_))
420	474	return res
421	475
422	476	class PropertyValue_ReferenceValue(ProtocolBuffer.ProtocolMessage):
…
827	881	kUserValuenickname = 11
828	882	kUserValuegaiaid = 18
829	883	kUserValueobfuscated_gaiaid = 19
	884	kUserValuefederated_identity = 21
	885	kUserValuefederated_provider = 22
830	886	kReferenceValueGroup = 12
831	887	kReferenceValueapp = 13
832	888	kReferenceValuename_space = 20
…
857	913	18: "gaiaid",
858	914	19: "obfuscated_gaiaid",
859	915	20: "name_space",
860		~~}, 20)~~
	916	21: "federated_identity",
	917	22: "federated_provider",
	918	}, 22)
861	919
862	920	_TYPES = _BuildTagLookupTable({
863	921	0: ProtocolBuffer.Encoder.NUMERIC,
…
881	939	18: ProtocolBuffer.Encoder.NUMERIC,
882	940	19: ProtocolBuffer.Encoder.STRING,
883	941	20: ProtocolBuffer.Encoder.STRING,
884		~~}, 20, ProtocolBuffer.Encoder.MAX_TYPE)~~
	942	21: ProtocolBuffer.Encoder.STRING,
	943	22: ProtocolBuffer.Encoder.STRING,
	944	}, 22, ProtocolBuffer.Encoder.MAX_TYPE)
885	945
886	946	_STYLE = """"""
887	947	_STYLE_CONTENT_TYPE = """"""
…
1525	1585	gaiaid_ = 0
1526	1586	has_obfuscated_gaiaid_ = 0
1527	1587	obfuscated_gaiaid_ = ""
	1588	has_federated_identity_ = 0
	1589	federated_identity_ = ""
	1590	has_federated_provider_ = 0
	1591	federated_provider_ = ""
1528	1592
1529	1593	def __init__(self, contents=None):
1530	1594	if contents is not None: self.MergeFromString(contents)
…
1594	1658
1595	1659	def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_
1596	1660
	1661	def federated_identity(self): return self.federated_identity_
	1662
	1663	def set_federated_identity(self, x):
	1664	self.has_federated_identity_ = 1
	1665	self.federated_identity_ = x
	1666
	1667	def clear_federated_identity(self):
	1668	if self.has_federated_identity_:
	1669	self.has_federated_identity_ = 0
	1670	self.federated_identity_ = ""
	1671
	1672	def has_federated_identity(self): return self.has_federated_identity_
	1673
	1674	def federated_provider(self): return self.federated_provider_
	1675
	1676	def set_federated_provider(self, x):
	1677	self.has_federated_provider_ = 1
	1678	self.federated_provider_ = x
	1679
	1680	def clear_federated_provider(self):
	1681	if self.has_federated_provider_:
	1682	self.has_federated_provider_ = 0
	1683	self.federated_provider_ = ""
	1684
	1685	def has_federated_provider(self): return self.has_federated_provider_
	1686
1597	1687
1598	1688	def MergeFrom(self, x):
1599	1689	assert x is not self
…
1602	1692	if (x.has_nickname()): self.set_nickname(x.nickname())
1603	1693	if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid())
1604	1694	if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid())
	1695	if (x.has_federated_identity()): self.set_federated_identity(x.federated_identity())
	1696	if (x.has_federated_provider()): self.set_federated_provider(x.federated_provider())
1605	1697
1606	1698	def Equals(self, x):
1607	1699	if x is self: return 1
…
1615	1707	if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0
1616	1708	if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0
1617	1709	if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0
	1710	if self.has_federated_identity_ != x.has_federated_identity_: return 0
	1711	if self.has_federated_identity_ and self.federated_identity_ != x.federated_identity_: return 0
	1712	if self.has_federated_provider_ != x.has_federated_provider_: return 0
	1713	if self.has_federated_provider_ and self.federated_provider_ != x.federated_provider_: return 0
1618	1714	return 1
1619	1715
1620	1716	def IsInitialized(self, debug_strs=None):
…
1640	1736	if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_))
1641	1737	n += self.lengthVarInt64(self.gaiaid_)
1642	1738	if (self.has_obfuscated_gaiaid_): n += 1 + self.lengthString(len(self.obfuscated_gaiaid_))
	1739	if (self.has_federated_identity_): n += 1 + self.lengthString(len(self.federated_identity_))
	1740	if (self.has_federated_provider_): n += 1 + self.lengthString(len(self.federated_provider_))
1643	1741	return n + 3
1644	1742
1645	1743	def Clear(self):
…
1648	1746	self.clear_nickname()
1649	1747	self.clear_gaiaid()
1650	1748	self.clear_obfuscated_gaiaid()
	1749	self.clear_federated_identity()
	1750	self.clear_federated_provider()
1651	1751
1652	1752	def OutputUnchecked(self, out):
1653	1753	out.putVarInt32(10)
…
1662	1762	if (self.has_obfuscated_gaiaid_):
1663	1763	out.putVarInt32(42)
1664	1764	out.putPrefixedString(self.obfuscated_gaiaid_)
	1765	if (self.has_federated_identity_):
	1766	out.putVarInt32(50)
	1767	out.putPrefixedString(self.federated_identity_)
	1768	if (self.has_federated_provider_):
	1769	out.putVarInt32(58)
	1770	out.putPrefixedString(self.federated_provider_)
1665	1771
1666	1772	def TryMerge(self, d):
1667	1773	while d.avail() > 0:
…
1681	1787	if tt == 42:
1682	1788	self.set_obfuscated_gaiaid(d.getPrefixedString())
1683	1789	continue
	1790	if tt == 50:
	1791	self.set_federated_identity(d.getPrefixedString())
	1792	continue
	1793	if tt == 58:
	1794	self.set_federated_provider(d.getPrefixedString())
	1795	continue
1684	1796	if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
1685	1797	d.skipData(tt)
1686	1798
…
1692	1804	if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_))
1693	1805	if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_))
1694	1806	if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_))
	1807	if self.has_federated_identity_: res+=prefix+("federated_identity: %s\n" % self.DebugFormatString(self.federated_identity_))
	1808	if self.has_federated_provider_: res+=prefix+("federated_provider: %s\n" % self.DebugFormatString(self.federated_provider_))
1695	1809	return res
1696	1810
1697	1811
…
1703	1817	knickname = 3
1704	1818	kgaiaid = 4
1705	1819	kobfuscated_gaiaid = 5
	1820	kfederated_identity = 6
	1821	kfederated_provider = 7
1706	1822
1707	1823	_TEXT = _BuildTagLookupTable({
1708	1824	0: "ErrorCode",
…
1711	1827	3: "nickname",
1712	1828	4: "gaiaid",
1713	1829	5: "obfuscated_gaiaid",
1714		~~}, 5)~~
	1830	6: "federated_identity",
	1831	7: "federated_provider",
	1832	}, 7)
1715	1833
1716	1834	_TYPES = _BuildTagLookupTable({
1717	1835	0: ProtocolBuffer.Encoder.NUMERIC,
…
1720	1838	3: ProtocolBuffer.Encoder.STRING,
1721	1839	4: ProtocolBuffer.Encoder.NUMERIC,
1722	1840	5: ProtocolBuffer.Encoder.STRING,
1723		~~}, 5, ProtocolBuffer.Encoder.MAX_TYPE)~~
	1841	6: ProtocolBuffer.Encoder.STRING,
	1842	7: ProtocolBuffer.Encoder.STRING,
	1843	}, 7, ProtocolBuffer.Encoder.MAX_TYPE)
1724	1844
1725	1845	_STYLE = """"""
1726	1846	_STYLE_CONTENT_TYPE = """"""

google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_config.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_config.py [2010-05-13 06:32:47.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Bulkloader Config Parser and runner.
	19
	20	A library to read bulkloader yaml configs.
	21	The code to interface between the bulkloader tool and the various connectors
	22	and conversions.
	23	"""
	24
	25
	26
	27
	28	import copy
	29	import os
	30	import sys
	31
	32	from google.appengine.api import datastore
	33	from google.appengine.ext.bulkload import bulkloader_errors
	34	from google.appengine.ext.bulkload import bulkloader_parser
	35	from google.appengine.ext.bulkload import csv_connector
	36	from google.appengine.ext.bulkload import simpletext_connector
	37	from google.appengine.ext.bulkload import simplexml_connector
	38
	39
	40	CONNECTOR_FACTORIES = {
	41	'csv': csv_connector.CsvConnector.create_from_options,
	42	'simplexml': simplexml_connector.SimpleXmlConnector.create_from_options,
	43	'simpletext': simpletext_connector.SimpleTextConnector.create_from_options,
	44	}
	45
	46
	47	class BulkloadState(object):
	48	"""Encapsulates state which is passed to other methods used in bulk loading.
	49
	50	It is optionally passed to import/export transform functions.
	51	It is passed to connector objects.
	52
	53	Properties:
	54	filename: The filename flag passed on the command line.
	55	loader_opts: The loader_opts flag passed on the command line.
	56	exporter_opts: The exporter_opts flag passed on the command line.
	57	current_instance: The current entity or model instance.
	58	current_entity: On export, the current entity instance.
	59	current_dictionary: The current input or output dictionary.
	60	"""
	61
	62	def __init__(self):
	63	self.filename = ''
	64	self.loader_opts = None
	65	self.exporter_opts = None
	66	self.current_instance = None
	67	self.current_entity = None
	68	self.current_dictionary = None
	69
	70
	71	def default_export_transform(value):
	72	"""A default export transform if nothing else is specified.
	73
	74	We assume most export connectors are string based, so a string cast is used.
	75	However, casting None to a string leads to 'None', so that's special cased.
	76
	77	Args:
	78	value: A value of some type.
	79
	80	Returns:
	81	unicode(value), or u'' if value is None
	82	"""
	83	if value is None:
	84	return u''
	85	else:
	86	return unicode(value)
	87
	88
	89	class DictConvertor(object):
	90	"""Convert a dict to an App Engine model instance or entity. And back.
	91
	92	The constructor takes a transformer spec representing a single transformer
	93	in a bulkloader.yaml.
	94
	95	The DictConvertor object has two public methods, dict_to_entity and
	96	entity_to_dict, which do the conversion between a neutral dictionary (the
	97	input/output of a connector) and an entity based on the spec.
	98
	99	Note that the model class may be used instead of an entity during the
	100	transform--this adds extra validation, etc, but also has a performance hit.
	101	"""
	102
	103	def __init__(self, transformer_spec):
	104	"""Constructor. See class docstring for more info.
	105
	106	Args:
	107	transformer_spec: A single transformer from a parsed bulkloader.yaml.
	108	This assumes that the transformer_spec is valid. It does not
	109	double check things like use_model_on_export requiring model.
	110	"""
	111	self._transformer_spec = transformer_spec
	112
	113	self._create_key = None
	114	for prop in self._transformer_spec.property_map:
	115	if prop.property == '__key__':
	116	self._create_key = prop
	117
	118	def dict_to_entity(self, input_dict, bulkload_state):
	119	"""Transform the dict to a model or entity instance(s).
	120
	121	Args:
	122	input_dict: Neutral input dictionary describing a single input record.
	123	bulkload_state: bulkload_state object describing the state.
	124
	125	Returns:
	126	Entity or model instance, or collection of entity or model instances,
	127	to be uploaded.
	128	"""
	129	bulkload_state_copy = copy.copy(bulkload_state)
	130	bulkload_state_copy.current_dictionary = input_dict
	131	instance = self.__create_instance(input_dict, bulkload_state_copy)
	132	bulkload_state_copy.current_instance = instance
	133	self.__run_import_transforms(input_dict, instance, bulkload_state_copy)
	134	if self._transformer_spec.post_import_function:
	135	post_map_instance = self._transformer_spec.post_import_function(
	136	input_dict, instance, bulkload_state_copy)
	137	return post_map_instance
	138	return instance
	139
	140	def entity_to_dict(self, entity, bulkload_state):
	141	"""Transform the entity to a dict, possibly via a model.
	142
	143	Args:
	144	entity: An entity.
	145	bulkload_state: bulkload_state object describing the global state.
	146
	147	Returns:
	148	A neutral output dictionary describing the record to write to the
	149	output.
	150	In the future this may return zero or multiple output dictionaries.
	151	"""
	152	if self._transformer_spec.use_model_on_export:
	153	instance = self._transformer_spec.model.from_entity(entity)
	154	else:
	155	instance = entity
	156
	157	export_dict = {}
	158	bulkload_state.current_entity = entity
	159	bulkload_state.current_instance = instance
	160	bulkload_state.current_dictionary = export_dict
	161	self.__run_export_transforms(instance, export_dict, bulkload_state)
	162	if self._transformer_spec.post_export_function:
	163	post_export_result = self._transformer_spec.post_export_function(
	164	instance, export_dict, bulkload_state)
	165	return post_export_result
	166	return export_dict
	167
	168	def __dict_to_prop(self, transform, input_dict, bulkload_state):
	169	"""Handle a single property on import.
	170
	171	Args:
	172	transform: The transform spec for this property.
	173	input_dict: Neutral input dictionary describing a single input record.
	174	bulkload_state: bulkload_state object describing the global state.
	175
	176	Returns:
	177	The value for this particular property.
	178	"""
	179	if transform.import_template:
	180	value = transform.import_template % input_dict
	181	else:
	182	value = input_dict.get(transform.external_name)
	183
	184	if transform.import_transform:
	185	if transform.import_transform.supports_bulkload_state:
	186	value = transform.import_transform(value, bulkload_state=bulkload_state)
	187	else:
	188	value = transform.import_transform(value)
	189	return value
	190
	191	def __create_instance(self, input_dict, bulkload_state):
	192	"""Return a model instance or entity from an input_dict.
	193
	194	Args:
	195	input_dict: Neutral input dictionary describing a single input record.
	196	bulkload_state: bulkload_state object describing the global state.
	197
	198	Returns:
	199	Entity or model instance, or collection of entity or model instances,
	200	to be uploaded.
	201	"""
	202	key = None
	203	parent = None
	204	if self._create_key:
	205	key = self.__dict_to_prop(self._create_key, input_dict, bulkload_state)
	206	if isinstance(key, datastore.Key):
	207	if not key.name():
	208	raise bulkloader_errors.ErrorOnTransform(
	209	'Numeric keys are not supported on input at this time.')
	210	parent = key.parent()
	211	key = key.name()
	212
	213	if self._transformer_spec.model:
	214	instance = self._transformer_spec.model(key_name=key, parent=parent)
	215	else:
	216	instance = datastore.Entity(self._transformer_spec.kind,
	217	parent=parent, name=key)
	218	return instance
	219
	220	def __run_import_transforms(self, input_dict, instance, bulkload_state):
	221	"""Fill in a single entity or model instance from an input_dict.
	222
	223	Args:
	224	input_dict: Input dict from the connector object.
	225	instance: Entity or model instance to fill in.
	226	bulkload_state: Passed bulkload state.
	227	"""
	228
	229	for transform in self._transformer_spec.property_map:
	230	if transform.property == '__key__':
	231	continue
	232
	233	value = self.__dict_to_prop(transform, input_dict, bulkload_state)
	234	if self._transformer_spec.model:
	235	setattr(instance, transform.property, value)
	236	else:
	237	instance[transform.property] = value
	238
	239	def __prop_to_dict(self, value, property_name, transform, export_dict,
	240	bulkload_state):
	241	"""Transform a single export-side field value to dict property.
	242
	243	Args:
	244	value: Value from the entity or model instance.
	245	property_name: Name of the value in the entity or model instance.
	246	transform: Transform property, either an ExportEntry or PropertyEntry
	247	export_dict: output dictionary.
	248	bulkload_state: Passed bulkload state.
	249
	250	Raises:
	251	ErrorOnTransform, encapsulating an error encountered during the transform.
	252	"""
	253	if transform.export_transform:
	254	try:
	255	if transform.export_transform.supports_bulkload_state:
	256	transformed_value = transform.export_transform(
	257	value, bulkload_state=bulkload_state)
	258	else:
	259	transformed_value = transform.export_transform(value)
	260	except Exception, err:
	261	raise bulkloader_errors.ErrorOnTransform(
	262	'Error on transform. '
	263	'Property: %s External Name: %s. Code: %s Details: %s' %
	264	(property_name, transform.external_name, transform.export_transform,
	265	err))
	266	else:
	267	transformed_value = default_export_transform(value)
	268	export_dict[transform.external_name] = transformed_value
	269
	270	def __run_export_transforms(self, instance, export_dict, bulkload_state):
	271	"""Fill in export_dict for an entity or model instance.
	272
	273	Args:
	274	instance: Entity or model instance
	275	export_dict: output dictionary.
	276	bulkload_state: Passed bulkload state.
	277	"""
	278	for transform in self._transformer_spec.property_map:
	279	if transform.property == '__key__':
	280	value = instance.key()
	281	elif self._transformer_spec.use_model_on_export:
	282	value = getattr(instance, transform.property, transform.default_value)
	283	else:
	284	value = instance.get(transform.property, transform.default_value)
	285
	286	if transform.export:
	287	for prop in transform.export:
	288	self.__prop_to_dict(value, transform.property, prop, export_dict,
	289	bulkload_state)
	290	elif transform.external_name:
	291	self.__prop_to_dict(value, transform.property, transform, export_dict,
	292	bulkload_state)
	293
	294
	295	class GenericImporter(object):
	296	"""Generic Bulkloader import class for input->dict->model transformation.
	297
	298	The bulkloader will call generate_records and create_entity, and
	299	we'll delegate those to the passed in methods.
	300	"""
	301
	302	def __init__(self, import_record_iterator, dict_to_entity, name):
	303	"""Constructor.
	304
	305	Args:
	306	import_record_iterator: Method which yields neutral dictionaries.
	307	dict_to_entity: Method dict_to_entity(input_dict) returns model or entity
	308	instance(s).
	309	name: Name to register with the bulkloader importers (as 'kind').
	310	"""
	311	self.import_record_iterator = import_record_iterator
	312	self.dict_to_entity = dict_to_entity
	313	self.kind = name
	314	self.bulkload_state = BulkloadState()
	315
	316	def get_high_ids(self):
	317	"""Required as part of the bulkloader Loader interface.
	318
	319	At the moment, this is not actually used by the bulkloader for import, as
	320	import does not currently support specifying numeric ids for keys.
	321	(Unspecified keys will become autogenerated ids.)
	322
	323	Returns:
	324	dict {ancestor_path : {kind : id}} of high id values, curently always {}.
	325	"""
	326	return {}
	327
	328	def initialize(self, filename, loader_opts):
	329	"""Performs initialization. Merely records the values for later use.
	330
	331	Args:
	332	filename: The string given as the --filename flag argument.
	333	loader_opts: The string given as the --loader_opts flag argument.
	334	"""
	335
	336	self.bulkload_state.loader_opts = loader_opts
	337	self.bulkload_state.filename = filename
	338
	339	def finalize(self):
	340	"""Performs finalization actions after the upload completes."""
	341	pass
	342
	343	def generate_records(self, filename):
	344	"""Iterator yielding neutral dictionaries from the connector object.
	345
	346	Args:
	347	filename: Filename argument passed in on the command line.
	348
	349	Returns:
	350	Iterator yielding neutral dictionaries, later passed to create_entity.
	351	"""
	352	return self.import_record_iterator(filename, self.bulkload_state)
	353
	354	def generate_key(self, line_number, unused_values):
	355	"""Bulkloader method to generate keys, mostly unused here.
	356
	357	This is called by the bulkloader just before it calls create_entity. The
	358	line_number is returned to be passed to the record dict, but otherwise
	359	unused.
	360
	361	Args:
	362	line_number: Record number from the bulkloader.
	363	unused_values: Neutral dict from generate_records; unused.
	364
	365	Returns:
	366	line_number for use later on.
	367	"""
	368	return line_number
	369
	370	def create_entity(self, values, key_name=None, parent=None):
	371	"""Creates entity/entities from input values via the dict_to_entity method.
	372
	373	Args:
	374	values: Neutral dict from generate_records.
	375	key_name: record number from generate_key.
	376	parent: Always None in this implementation of a Loader.
	377
	378	Returns:
	379	Entity or model instance, or collection of entity or model instances,
	380	to be uploaded.
	381	"""
	382
	383	input_dict = values
	384	input_dict['__record_number__'] = key_name
	385	return self.dict_to_entity(input_dict, self.bulkload_state)
	386
	387
	388	class GenericExporter(object):
	389	"""Implements bulkloader.Exporter interface and delegates.
	390
	391	This will delegate to the passed in entity_to_dict method and the
	392	methods on the export_recorder which are in the ConnectorInterface.
	393	"""
	394
	395	def __init__(self, export_recorder, entity_to_dict, kind,
	396	sort_key_from_entity):
	397	"""Constructor.
	398
	399	Args:
	400	export_recorder: Object which writes results, an implementation of
	401	ConnectorInterface.
	402	entity_to_dict: Method which converts a single entity to a neutral dict.
	403	kind: Kind to identify this object to the bulkloader.
	404	sort_key_from_entity: Optional method to return a sort key for each
	405	entity. This key will be used to sort the downloaded entities before
	406	passing them to eneity_to_dict.
	407	"""
	408	self.export_recorder = export_recorder
	409	self.entity_to_dict = entity_to_dict
	410	self.kind = kind
	411	self.sort_key_from_entity = sort_key_from_entity
	412	self.calculate_sort_key_from_entity = bool(sort_key_from_entity)
	413	self.bulkload_state = BulkloadState()
	414
	415	def initialize(self, filename, exporter_opts):
	416	"""Performs initialization and validation of the output file.
	417
	418	Args:
	419	filename: The string given as the --filename flag argument.
	420	exporter_opts: The string given as the --exporter_opts flag argument.
	421	"""
	422	self.bulkload_state.filename = filename
	423	self.bulkload_state.exporter_opts = exporter_opts
	424	self.export_recorder.initialize_export(filename, self.bulkload_state)
	425
	426	def output_entities(self, entity_iterator):
	427	"""Outputs the downloaded entities.
	428
	429	Args:
	430	entity_iterator: An iterator that yields the downloaded entities
	431	in sorted order.
	432	"""
	433	for entity in entity_iterator:
	434	output_dict = self.entity_to_dict(entity, self.bulkload_state)
	435	if output_dict:
	436	self.export_recorder.write_dict(output_dict)
	437
	438	def finalize(self):
	439	"""Performs finalization actions after the download completes."""
	440	self.export_recorder.finalize_export()
	441
	442
	443	def create_transformer_classes(transformer_spec, config_globals):
	444	"""Create an importer and exporter class from a transformer spec.
	445
	446	Args:
	447	transformer_spec: A bulkloader_parser.TransformerEntry.
	448	config_globals: Dict to use to reference globals for code in the config.
	449
	450	Raises:
	451	InvalidConfig: when the config is invalid.
	452
	453	Returns:
	454	Tuple, (importer class, exporter class), each which is in turn a wrapper
	455	for the GenericImporter/GenericExporter class using a DictConvertor object
	456	configured as per the transformer_spec.
	457	"""
	458	if transformer_spec.connector in CONNECTOR_FACTORIES:
	459	connector_factory = CONNECTOR_FACTORIES[transformer_spec.connector]
	460	elif config_globals and '.' in transformer_spec.connector:
	461	try:
	462	connector_factory = eval(transformer_spec.connector, config_globals)
	463	except (NameError, AttributeError):
	464	raise bulkloader_errors.InvalidConfiguration(
	465	'Invalid connector specified for name=%s. Could not evaluate %s.' %
	466	(transformer_spec.name, transformer_spec.connector))
	467	else:
	468	raise bulkloader_errors.InvalidConfiguration(
	469	'Invalid connector specified for name=%s. Must be either a built in '
	470	'connector ("%s") or a factory method in a module imported via '
	471	'python_preamble.' %
	472	(transformer_spec.name, '", "'.join(CONNECTOR_FACTORIES)))
	473	options = {}
	474	if transformer_spec.connector_options:
	475	options = transformer_spec.connector_options.ToDict()
	476
	477	try:
	478	connector_object = connector_factory(options, transformer_spec.name)
	479	except TypeError:
	480	raise bulkloader_errors.InvalidConfiguration(
	481	'Invalid connector specified for name=%s. Could not initialize %s.' %
	482	(transformer_spec.name, transformer_spec.connector))
	483
	484
	485	dict_to_model_object = DictConvertor(transformer_spec)
	486
	487	class ImporterClass(GenericImporter):
	488	"""Class to pass to the bulkloader, wraps the specificed configuration."""
	489
	490	def __init__(self):
	491	super(self.__class__, self).__init__(
	492	connector_object.generate_import_record,
	493	dict_to_model_object.dict_to_entity,
	494	transformer_spec.name)
	495	importer_class = ImporterClass
	496
	497	class ExporterClass(GenericExporter):
	498	"""Class to pass to the bulkloader, wraps the specificed configuration."""
	499
	500	def __init__(self):
	501	super(self.__class__, self).__init__(
	502	connector_object,
	503	dict_to_model_object.entity_to_dict,
	504	transformer_spec.kind,
	505	transformer_spec.sort_key_from_entity)
	506	exporter_class = ExporterClass
	507
	508	return importer_class, exporter_class
	509
	510
	511	def load_config_from_stream(stream):
	512	"""Parse a bulkloader.yaml file into bulkloader loader classes.
	513
	514	Args:
	515	stream: A stream containing bulkloader.yaml data.
	516
	517	Returns:
	518	importer_classes, exporter_classes: Constructors suitable to pass to the
	519	bulkloader.
	520	"""
	521	config_globals = {}
	522	config = bulkloader_parser.load_config(stream, config_globals)
	523	importer_classes = []
	524	exporter_classes = []
	525	for transformer in config.transformers:
	526	importer, exporter = create_transformer_classes(transformer, config_globals)
	527	if importer:
	528	importer_classes.append(importer)
	529	if exporter:
	530	exporter_classes.append(exporter)
	531
	532	return importer_classes, exporter_classes
	533
	534
	535	def load_config(filename, update_path=True):
	536	"""Load a configuration file and create importer and exporter classes.
	537
	538	Args:
	539	filename: Filename of bulkloader.yaml.
	540	update_path: Should sys.path be extended to include the path of filename?
	541
	542	Returns:
	543	Tuple, (importer classes, exporter classes) based on the transformers
	544	specified in the file.
	545	"""
	546
	547	if update_path:
	548	sys.path.append(os.path.abspath(os.path.dirname(os.path.abspath(filename))))
	549	stream = file(filename, 'r')
	550	try:
	551	return load_config_from_stream(stream)
	552	finally:
	553	stream.close()

google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_errors.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_errors.py [2010-05-13 06:32:48.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Exceptions raised by bulkloader methods."""
	19
	20
	21
	22
	23	class Error(Exception):
	24	"""Base bulkloader error type."""
	25
	26
	27	class ErrorOnTransform(Error):
	28	"""An exception was raised during this transform."""
	29
	30
	31	class InvalidConfiguration(Error):
	32	"""The configuration is invalid."""
	33
	34
	35	class InvalidCodeInConfiguration(Error):
	36	"""A code or lambda statement in the configuration could not be evaulated."""
	37
	38
	39	class InvalidExportData(Error):
	40	"""The export data cannot be written using this connector object."""

google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_parser.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_parser.py [2010-05-13 06:32:48.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Bulkloader Config Parser and runner.
	19
	20	A library to read bulkloader yaml configs. Returns a BulkloaderEntry object
	21	which describes the bulkloader.yaml in object form, including some additional
	22	parsing of things like Python lambdas.
	23	"""
	24
	25
	26
	27
	28	import inspect
	29	import sys
	30
	31	from google.appengine.api import validation
	32	from google.appengine.api import yaml_builder
	33	from google.appengine.api import yaml_listener
	34	from google.appengine.api import yaml_object
	35
	36	from google.appengine.ext.bulkload import bulkloader_errors
	37
	38
	39	_global_temp_globals = None
	40
	41
	42	class EvaluatedCallable(validation.Validator):
	43	"""Validates that a string evaluates to a Python callable.
	44
	45	Calls eval at validation time and stores the results as a ParsedMethod object.
	46	The ParsedMethod object can be used as a string (original value) or callable
	47	(parsed method). It also exposes supports_bulkload_state if the callable has
	48	a kwarg called 'bulkload_state', which is used to determine how to call
	49	the *_transform methods.
	50	"""
	51
	52	class ParsedMethod(object):
	53	"""Wrap the string, the eval'd method, and supports_bulkload_state."""
	54
	55	def __init__(self, value, key):
	56	"""Initialze internal state.
	57
	58	Eval the string value and save the result.
	59
	60	Args:
	61	value: String to compile as a regular expression.
	62	key: The YAML field name.
	63
	64	Raises:
	65	InvalidCodeInConfiguration: if the code could not be evaluated, or
	66	the evalauted method is not callable.
	67	"""
	68	self.value = value
	69	try:
	70	self.method = eval(value, _global_temp_globals)
	71	except Exception, err:
	72	raise bulkloader_errors.InvalidCodeInConfiguration(
	73	'Invalid code for %s. Code: "%s". Details: %s' % (key, value, err))
	74	if not callable(self.method):
	75	raise bulkloader_errors.InvalidCodeInConfiguration(
	76	'Code for %s did not return a callable. Code: "%s".' %
	77	(key, value))
	78
	79	self.supports_bulkload_state = False
	80	try:
	81	argspec = inspect.getargspec(self.method)
	82	if 'bulkload_state' in argspec[0]:
	83	self.supports_bulkload_state = True
	84	except TypeError:
	85	pass
	86
	87	def __str__(self):
	88	"""Return a string representation of the method: the original string."""
	89	return self.value
	90
	91	def __call__(self, args, *kwargs):
	92	"""Call the method."""
	93	return self.method(args, *kwargs)
	94
	95	def __init__(self):
	96	"""Initialize EvaluatedCallable validator."""
	97	super(EvaluatedCallable, self).__init__()
	98
	99	def Validate(self, value, key):
	100	"""Validates that the string compiles as a Python callable.
	101
	102	Args:
	103	value: String to compile as a regular expression.
	104	key: The YAML field name.
	105
	106	Returns:
	107	Value wrapped in an object with properties 'value' and 'fn'.
	108
	109	Raises:
	110	InvalidCodeInConfiguration when value does not compile.
	111	"""
	112	if isinstance(value, self.ParsedMethod):
	113	return value
	114	else:
	115	return self.ParsedMethod(value, key)
	116
	117	def ToValue(self, value):
	118	"""Returns the code string for this value."""
	119	return value.value
	120
	121
	122	OPTIONAL_EVALUATED_CALLABLE = validation.Optional(EvaluatedCallable())
	123
	124
	125	class ConnectorSubOptions(validation.Validated):
	126	"""Connector options."""
	127
	128	ATTRIBUTES = {
	129	'delimiter': validation.Optional(validation.TYPE_STR),
	130	'dialect': validation.Optional(validation.TYPE_STR),
	131	}
	132
	133
	134	class ConnectorOptions(validation.Validated):
	135	"""Connector options."""
	136
	137	ATTRIBUTES = {
	138	'column_list':
	139	validation.Optional(validation.Repeated(validation.TYPE_STR)),
	140	'columns': validation.Optional(validation.TYPE_STR),
	141	'encoding': validation.Optional(validation.TYPE_STR),
	142	'epilog': validation.Optional(validation.TYPE_STR),
	143	'export_options': validation.Optional(ConnectorSubOptions),
	144	'import_options': validation.Optional(ConnectorSubOptions),
	145	'mode': validation.Optional(validation.TYPE_STR),
	146	'prolog': validation.Optional(validation.TYPE_STR),
	147	'style': validation.Optional(validation.TYPE_STR),
	148	'template': validation.Optional(validation.TYPE_STR),
	149	'xpath_to_nodes': validation.Optional(validation.TYPE_STR),
	150	'print_export_header_row': validation.Optional(validation.TYPE_BOOL),
	151	'skip_import_header_row': validation.Optional(validation.TYPE_BOOL),
	152	}
	153
	154	def CheckInitialized(self):
	155	"""Post-loading 'validation'. Really used to fix up yaml hackyness."""
	156	super(ConnectorOptions, self).CheckInitialized()
	157
	158	if self.column_list:
	159	self.column_list = [str(column) for column in self.column_list]
	160
	161
	162	class ExportEntry(validation.Validated):
	163	"""Describes the optional export transform for a single property."""
	164
	165	ATTRIBUTES = {
	166	'external_name': validation.Optional(validation.TYPE_STR),
	167	'export_transform': OPTIONAL_EVALUATED_CALLABLE,
	168	}
	169
	170
	171	class PropertyEntry(validation.Validated):
	172	"""Describes the transform for a single property."""
	173
	174	ATTRIBUTES = {
	175	'property': validation.Type(str),
	176	'import_transform': OPTIONAL_EVALUATED_CALLABLE,
	177	'import_template': validation.Optional(validation.TYPE_STR),
	178	'default_value': validation.Optional(validation.TYPE_STR),
	179	'export': validation.Optional(validation.Repeated(ExportEntry)),
	180	}
	181	ATTRIBUTES.update(ExportEntry.ATTRIBUTES)
	182
	183	def CheckInitialized(self):
	184	"""Check that all required (combinations) of fields are set.
	185
	186	Also fills in computed properties.
	187
	188	Raises:
	189	InvalidConfiguration: If the config is invalid.
	190	"""
	191	super(PropertyEntry, self).CheckInitialized()
	192
	193	if not (self.external_name or self.import_template or self.export):
	194	raise bulkloader_errors.InvalidConfiguration(
	195	'Neither external_name nor import_template nor export specified for '
	196	'property %s.' % self.property)
	197
	198
	199	class TransformerEntry(validation.Validated):
	200	"""Describes the transform for an entity (or model) kind."""
	201
	202	ATTRIBUTES = {
	203	'name': validation.Optional(validation.TYPE_STR),
	204	'kind': validation.Optional(validation.TYPE_STR),
	205	'model': OPTIONAL_EVALUATED_CALLABLE,
	206	'connector': validation.TYPE_STR,
	207	'connector_options': validation.Optional(ConnectorOptions, {}),
	208	'use_model_on_export': validation.Optional(validation.TYPE_BOOL),
	209	'sort_key_from_entity': OPTIONAL_EVALUATED_CALLABLE,
	210	'post_import_function': OPTIONAL_EVALUATED_CALLABLE,
	211	'post_export_function': OPTIONAL_EVALUATED_CALLABLE,
	212	'property_map': validation.Repeated(PropertyEntry, default=[]),
	213	}
	214
	215	def CheckInitialized(self):
	216	"""Check that all required (combinations) of fields are set.
	217
	218	Also fills in computed properties.
	219
	220	Raises:
	221	InvalidConfiguration: if the config is invalid.
	222	"""
	223	if not self.kind and not self.model:
	224	raise bulkloader_errors.InvalidConfiguration(
	225	'Neither kind nor model specified for transformer.')
	226	if self.kind and self.model:
	227	raise bulkloader_errors.InvalidConfiguration(
	228	'Both kind and model specified for transformer.')
	229
	230	if self.model:
	231	self.kind = self.model().kind()
	232	else:
	233	if self.use_model_on_export:
	234	raise bulkloader_errors.InvalidConfiguration(
	235	'No model class specified but use_model_on_export is true.')
	236	if not self.name:
	237	self.name = self.kind
	238
	239	if not self.connector:
	240	raise bulkloader_errors.InvalidConfiguration('No connector specified.')
	241
	242	property_names = set()
	243	for prop in self.property_map:
	244	if prop.property in property_names:
	245	raise bulkloader_errors.InvalidConfiguration(
	246	'Duplicate property specified for property %s in transform %s' %
	247	(prop.property, self.name))
	248	property_names.add(prop.property)
	249
	250
	251	class PythonPreambleEntry(validation.Validated):
	252	"""Python modules to import at initialization time, typically models."""
	253
	254	ATTRIBUTES = {'import': validation.TYPE_STR,
	255	'as': validation.Optional(validation.TYPE_STR),
	256	}
	257
	258	def CheckInitialized(self):
	259	"""Check that all required fields are set, and update global state.
	260
	261	The imports specified in the preamble are imported at this time.
	262	"""
	263	python_import = getattr(self, 'import')
	264	topname = python_import.split('.')[0]
	265	module_name = getattr(self, 'as')
	266	if not module_name:
	267	module_name = python_import.split('.')[-1]
	268
	269	__import__(python_import, _global_temp_globals)
	270	_global_temp_globals[topname] = sys.modules[topname]
	271	_global_temp_globals[module_name] = sys.modules[python_import]
	272
	273
	274	class BulkloaderEntry(validation.Validated):
	275	"""Root of the bulkloader configuration."""
	276
	277	ATTRIBUTES = {
	278	'python_preamble':
	279	validation.Optional(validation.Repeated(PythonPreambleEntry)),
	280	'transformers': validation.Repeated(TransformerEntry),
	281	}
	282
	283
	284	def load_config(stream, config_globals):
	285	"""Load a configuration file and generate importer and exporter classes.
	286
	287	Args:
	288	stream: Stream containing config YAML.
	289	config_globals: Dict to use to reference globals for code in the config.
	290
	291	Returns:
	292	BulkloaderEntry
	293
	294	Raises:
	295	InvalidConfiguration: If the config is invalid.
	296	"""
	297	builder = yaml_object.ObjectBuilder(BulkloaderEntry)
	298	handler = yaml_builder.BuilderHandler(builder)
	299	listener = yaml_listener.EventListener(handler)
	300	global _global_temp_globals
	301	_global_temp_globals = config_globals
	302	try:
	303	listener.Parse(stream)
	304	finally:
	305	_global_temp_globals = None
	306
	307	bulkloader_infos = handler.GetResults()
	308	if len(bulkloader_infos) < 1:
	309	raise bulkloader_errors.InvalidConfiguration('No configuration specified.')
	310	if len(bulkloader_infos) > 1:
	311	raise bulkloader_errors.InvalidConfiguration(
	312	'Multiple sections in configuration.')
	313	bulkloader_info = bulkloader_infos[0]
	314	if not bulkloader_info.transformers:
	315	raise bulkloader_errors.InvalidConfiguration('No transformers specified.')
	316	return bulkloader_info

google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.py [2010-05-13 06:32:48.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Wizard to generate bulkloader configuration.
	19
	20	Helper functions to call from the bulkloader.yaml.
	21	The wizard is run by having bulkloader.py download datastore statistics
	22	(http://code.google.com/appengine/docs/python/datastore/stats.html ,
	23	specifically __Stat_PropertyType_PropertyName_Kind__) configured with
	24	bulkloader_wizard.yaml.
	25	"""
	26
	27
	28	PROPERTY_DUPE_WARNING = (
	29	' # Warning: This property is a duplicate, but with a different type.\n'
	30	' # TODO: Edit this transform so only one property with this name '
	31	'remains.\n')
	32
	33	KIND_PREAMBLE = """
	34	- kind: %(kind_name)s
	35	connector: # TODO: Choose a connector here: csv, simplexml, etc...
	36	connector_options:
	37	# TODO: Add connector options here--these are specific to each connector.
	38	property_map:
	39	- property: __key__
	40	external_name: key
	41	export_transform: transform.key_id_or_name_as_string
	42
	43	"""
	44
	45
	46	class StatPostTransform(object):
	47	"""Create text to insert between properties and filter out 'bad' properties.
	48
	49	This class is a callable post_export_function which saves state
	50	across multiple calls.
	51
	52	It uses this saved state to determine if each entity is the first entity seen
	53	of a new kind, a duplicate kind/propertyname entry, or just a new property
	54	in the current kind being processed.
	55
	56	It will suppress bad output by returning None for NULL property types and
	57	__private__ types (notably the stats themselves).
	58	"""
	59
	60	def __init__(self):
	61	"""Constructor.
	62
	63	Attributes:
	64	seen_properties: (kind, propertyname) -> number of times seen before. If
	65	seen more than once, this is a duplicate property for the kind.
	66	last_seen: Previous kind seen. If it changes, this is a new kind.
	67	"""
	68	self.seen_properties = {}
	69	self.last_seen = None
	70
	71	def __call__(self, instance, dictionary, bulkload_state):
	72	"""Implementation of StatPropertyTypePropertyNameKindPostExport.
	73
	74	See class docstring for more info.
	75
	76	Args:
	77	instance: Input, current entity being exported.
	78	dictionary: Output, dictionary created by property_map transforms.
	79	bulkload_state: Passed bulkload_state.
	80
	81	Returns:
	82	Dictionary--same object as passed in dictionary.
	83	"""
	84	kind_name = dictionary['kind_name']
	85	property_name = dictionary['property_name']
	86	property_type = dictionary['property_type']
	87
	88	if kind_name.startswith('__'):
	89	return None
	90	if property_type == 'NULL':
	91	return None
	92
	93	property_key = kind_name, property_name
	94	if kind_name != self.last_seen:
	95	self.last_seen = kind_name
	96	separator = KIND_PREAMBLE % dictionary
	97	elif property_key in self.seen_properties:
	98	separator = PROPERTY_DUPE_WARNING % dictionary
	99	else:
	100	separator = ''
	101	self.seen_properties[property_key] = (
	102	self.seen_properties.get(property_key, 0) + 1)
	103
	104	dictionary['separator'] = separator
	105	return dictionary
	106
	107
	108	TYPE_TO_TRANSFORM_MAP = {
	109	'Blob': ('transform.blobproperty_from_base64',
	110	'base64.b64encode'),
	111	'Boolean': ('transform.regexp_bool(\'true\', re.IGNORECASE)',
	112	None),
	113	'ByteString': ('transform.bytestring_from_base64', 'base64.b64encode'),
	114	'Category': ('db.Category', None),
	115	'Date/Time': ('transform.import_date_time(\'%Y-%m-%dT%H:%M:%S\')',
	116	'transform.export_date_time(\'%Y-%m-%dT%H:%M:%S\')'),
	117	'Email': ('db.Email', None),
	118	'Float': ('transform.none_if_empty(float)', None),
	119
	120	'Integer': ('transform.none_if_empty(int)', None),
	121	'Key': ('transform.create_foreign_key(\'TODO: fill in Kind name\')',
	122	'transform.key_id_or_name_as_string'),
	123	'Link': ('db.Link', None),
	124
	125	'PhoneNumber': ('db.PhoneNumber', None),
	126	'PostalAddress': ('db.PostalAddress', None),
	127	'Rating': ('transform.none_if_empty(db.Rating)', None),
	128	'String': (None, None),
	129	'Text': ('db.Text', None),
	130	'User': ('transform.none_if_empty(users.User) # Assumes email address',
	131	None),
	132	}
	133
	134
	135	def DatastoreTypeToTransforms(property_type):
	136	"""Return the import/export_transform lines for a datastore type.
	137
	138	Args:
	139	property_type: Property type from the KindPropertyNamePropertyTypeStat.
	140
	141	Returns:
	142	Strings for use in a bulkloader.yaml as transforms. This
	143	may be '' (no transform needed), or one or two lines with import_transform
	144	or export_transform.
	145	"""
	146	import_transform, export_transform = TYPE_TO_TRANSFORM_MAP.get(property_type,
	147	(None, None))
	148	transform = []
	149	if import_transform:
	150	transform.append(' import_transform: %s\n' % import_transform)
	151	if export_transform:
	152	transform.append(' export_transform: %s\n' % export_transform)
	153
	154	return ''.join(transform)

google_appengine_1.3.3/google/appengine/ext/bulkload/bulkloader_wizard.yaml [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/bulkloader_wizard.yaml [2010-05-13 06:32:47.000000000 +0900]

	1	# Transform description for generating bulkloader configuration by bulkloading
	2	# the datastore stats.
	3
	4	python_preamble:
	5	- import: base64
	6	- import: re
	7	- import: google.appengine.ext.bulkload.transform
	8	- import: google.appengine.ext.bulkload.bulkloader_wizard
	9	- import: google.appengine.api.datastore
	10	- import: google.appengine.api.users
	11
	12	transformers:
	13	- kind: __Stat_PropertyType_PropertyName_Kind__
	14	connector: simpletext
	15	connector_options:
	16	mode: nonewline
	17	prolog: \|
	18	# Autogenerated bulkloader.yaml file.
	19	# You're likely to have to do various edits to this file:
	20	# At a minimum address the items marked with TODO:
	21	# * Fill in connector and connector_options
	22	# * Review the property_map.
	23	# - Ensure the 'external_name' matches the name of your CSV column,
	24	# XML tag, etc.
	25	# - Check that __key__ property is what you want. Its value will become
	26	# the key name on import, and on export the value will be the Key
	27	# object. If you would like automatic key generation on import and
	28	# omitting the key on export, you can remove the entire __key__
	29	# property from the property map.
	30
	31	# If you have module(s) with your model classes, add them here. Also
	32	# change the kind properties to model_class.
	33	python_preamble:
	34	- import: google.appengine.ext.bulkload.transform
	35	- import: google.appengine.ext.db
	36	- import: re
	37	- import: base64
	38
	39	transformers:
	40	template: \|
	41	%(separator)s - property: %(property_name)s
	42	external_name: %(property_name)s
	43	# Type: %(property_type)s Stats: %(count)s properties of this type in this kind.
	44	%(transforms)s
	45	sort_key_from_entity:
	46	"lambda entity: '_'.join((entity.get('kind_name', ''),
	47	entity.get('property_name', ''),
	48	entity.get('property_type', '')))"
	49	property_map:
	50	- property: __key__
	51	external_name: key
	52	export_transform: datastore.Key.name
	53	- property: kind_name
	54	external_name: kind_name
	55	default_value: MISSING_KIND_NAME
	56	- property: property_name
	57	external_name: property_name
	58	- property: property_type
	59	external_name: property_type
	60	export:
	61	- external_name: property_type
	62	- external_name: transforms
	63	export_transform: bulkloader_wizard.DatastoreTypeToTransforms
	64	- property: bytes
	65	external_name: bytes
	66	- property: count
	67	external_name: count
	68	- property: timestamp
	69	external_name: timestamp
	70	export_transform: transform.export_date_time('%Y%m%dT%H:%M')
	71	post_export_function: bulkloader_wizard.StatPostTransform()

google_appengine_1.3.3/google/appengine/ext/bulkload/connector_interface.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/connector_interface.py [2010-05-13 06:32:47.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Bulkloader interfaces for the format reader/writers."""
	19
	20
	21
	22
	23
	24	class ConnectorInterface(object):
	25	"""Abstract base class describing the external Connector interface.
	26
	27	The External Connector interface describes the transition between an external
	28	data source, e.g. CSV file, XML file, or some sort of database interface, and
	29	the intermediate bulkloader format, which is a dictionary or similar
	30	structure representing the external transformation of the data.
	31
	32	On import, the generate_import_record generator is the only method called.
	33
	34	On export, the initialize_export method is called once, followed by a call
	35	to write_dict for each record, followed by a call to finalize_export.
	36
	37	The bulkload_state is a BulkloadState object from
	38	google.appengine.ext.bulkload.bulkload_config. The interesting properties
	39	to a Connector object are the loader_opts and exporter_opts, which are strings
	40	passed in from the bulkloader command line.
	41	"""
	42
	43	def generate_import_record(self, filename, bulkload_state):
	44	"""A function which returns an interator over dictionaries.
	45
	46	This is the only method used on import.
	47
	48	Args:
	49	filename: The --filename argument passed in on the bulkloader command
	50	line. This value is opaque to the bulkloader and thus could specify
	51	any sort of descriptor for your generator.
	52	bulkload_state: Passed in BulkloadConfig.BulkloadState object.
	53
	54	Returns:
	55	An iterator describing an individual record. Typically a dictionary,
	56	to be used with dict_to_model. Typically implemented as a generator.
	57	"""
	58	raise NotImplementedError
	59
	60	def initialize_export(self, filename, bulkload_state):
	61	"""Initialize the output file.
	62
	63	Args:
	64	filename: The string given as the --filename flag argument.
	65	bulkload_state: Passed in BulkloadConfig.BulkloadState object.
	66
	67	These values are opaque to the bulkloader and thus could specify
	68	any sort of descriptor for your exporter.
	69	"""
	70	raise NotImplementedError
	71
	72	def write_dict(self, dictionary):
	73	"""Write one record for the specified entity.
	74
	75	Args:
	76	dictionary: A post-transform dictionary.
	77	"""
	78	raise NotImplementedError
	79
	80	def finalize_export(self):
	81	"""Performs finalization actions after every record is written."""
	82	raise NotImplementedError
	83
	84
	85	def create_from_options(options, name='unknown'):
	86	"""Factory using an options dictionary.
	87
	88	This is frequently implemented as the constructor on the connector class,
	89	or a static or class method on the connector class.
	90
	91	Args:
	92	options: Parsed dictionary from yaml file, interpretation is up to the
	93	implementor of this class.
	94	name: Identifier of this transform to be used in error messages.
	95
	96	Returns:
	97	An object which implements the ConnectorInterface interface.
	98	"""
	99	raise NotImplementedError

google_appengine_1.3.3/google/appengine/ext/bulkload/csv_connector.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/csv_connector.py [2010-05-13 06:32:47.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	#
	17
	18	"""Bulkloader CSV reading and writing.
	19
	20	Handle the CSV format specified in a bulkloader.yaml file.
	21	"""
	22
	23
	24
	25
	26
	27	import codecs
	28	import cStringIO
	29	import csv
	30	import encodings
	31	import encodings.ascii
	32	import encodings.cp1252
	33	import encodings.latin_1
	34	import encodings.utf_8
	35
	36	from google.appengine.ext.bulkload import bulkloader_errors
	37	from google.appengine.ext.bulkload import connector_interface
	38
	39
	40	def utf8_recoder(stream, encoding):
	41	"""Generator that reads an encoded stream and reencodes to UTF-8."""
	42	for line in codecs.getreader(encoding)(stream):
	43	yield line.encode('utf-8')
	44
	45
	46	class UnicodeDictWriter(object):
	47	"""Based on UnicodeWriter in http://docs.python.org/library/csv.html."""
	48
	49	def __init__(self, stream, fieldnames, encoding='utf-8', **kwds):
	50	"""Initialzer.
	51
	52	Args:
	53	stream: Stream to write to.
	54	fieldnames: Fieldnames to pass to the DictWriter.
	55	encoding: Desired encoding.
	56	kwds: Additional arguments to pass to the DictWriter.
	57	"""
	58	writer = codecs.getwriter(encoding)
	59	if (writer is encodings.utf_8.StreamWriter or
	60	writer is encodings.ascii.StreamWriter or
	61	writer is encodings.latin_1.StreamWriter or
	62	writer is encodings.cp1252.StreamWriter):
	63	self.no_recoding = True
	64	self.encoder = codecs.getencoder(encoding)
	65	self.writer = csv.DictWriter(stream, fieldnames, **kwds)
	66	else:
	67	self.no_recoding = False
	68	self.encoder = codecs.getencoder('utf-8')
	69	self.queue = cStringIO.StringIO()
	70	self.writer = csv.DictWriter(self.queue, fieldnames, **kwds)
	71	self.stream = writer(stream)
	72
	73	def writerow(self, row):
	74	"""Wrap writerow method."""
	75	row_encoded = dict([(k, self.encoder(v)[0]) for (k, v) in row.iteritems()])
	76	self.writer.writerow(row_encoded)
	77	if self.no_recoding:
	78	return
	79
	80	data = self.queue.getvalue()
	81	data = data.decode('utf-8')
	82	self.stream.write(data)
	83	self.queue.truncate(0)
	84
	85
	86	class CsvConnector(connector_interface.ConnectorInterface):
	87	"""Read/write a (possibly encoded) CSV file."""
	88
	89	@classmethod
	90	def create_from_options(cls, options, name):
	91	"""Factory using an options dictionary.
	92
	93	Args:
	94	options: Dictionary of options:
	95	columns: 'from_header' or blank.
	96	column_list: overrides columns specifically.
	97	encoding: encoding of the file. e.g. 'utf-8' (default), 'windows-1252'.
	98	skip_import_header_row: True to ignore the header line on import.
	99	Defaults False, except must be True if columns=from_header.
	100	print_export_header_row: True to print a header line on export.
	101	Defaults to False except if columns=from_header.
	102	import_options: Other kwargs to pass in, like "dialect".
	103	export_options: Other kwargs to pass in, like "dialect".
	104	name: The name of this transformer, for use in error messages.
	105
	106	Returns:
	107	CsvConnector object described by the specified options.
	108
	109	Raises:
	110	InvalidConfiguration: If the config is invalid.
	111	"""
	112	column_list = options.get('column_list', None)
	113	columns = None
	114	if not column_list:
	115	columns = options.get('columns', 'from_header')
	116	if columns != 'from_header':
	117	raise bulkloader_errors.InvalidConfiguration(
	118	'CSV columns must be "from_header", or a column_list '
	119	'must be specified. (In transformer name %s.)' % name)
	120	csv_encoding = options.get('encoding', 'utf-8')
	121
	122
	123	skip_import_header_row = options.get('skip_import_header_row',
	124	columns == 'from_header')
	125	if columns == 'from_header' and not skip_import_header_row:
	126	raise bulkloader_errors.InvalidConfiguration(
	127	'When CSV columns are "from_header", the header row must always '
	128	'be skipped. (In transformer name %s.)' % name)
	129	print_export_header_row = options.get('print_export_header_row',
	130	columns == 'from_header')
	131	import_options = options.get('import_options', {})
	132	export_options = options.get('export_options', {})
	133	return cls(columns, column_list, skip_import_header_row,
	134	print_export_header_row, csv_encoding, import_options,
	135	export_options)
	136
	137	def __init__(self, columns, column_list, skip_import_header_row,
	138	print_export_header_row, csv_encoding=None,
	139	import_options=None, export_options=None):
	140	"""Initializer.
	141
	142	Args:
	143	columns: 'from_header' or blank
	144	column_list: overrides columns specifically.
	145	skip_import_header_row: True to ignore the header line on import.
	146	Defaults False, except must be True if columns=from_header.
	147	print_export_header_row: True to print a header line on export.
	148	Defaults to False except if columns=from_header.
	149	csv_encoding: encoding of the file.
	150	import_options: Other kwargs to pass in, like "dialect".
	151	export_options: Other kwargs to pass in, like "dialect".
	152	"""
	153
	154	self.columns = columns
	155	self.from_header = (columns == 'from_header')
	156	self.column_list = column_list
	157	self.skip_import_header_row = skip_import_header_row
	158	self.print_export_header_row = print_export_header_row
	159	self.csv_encoding = csv_encoding
	160	self.dict_generator = None
	161	self.output_stream = None
	162	self.csv_writer = None
	163	self.bulkload_state = None
	164	self.import_options = import_options or {}
	165	self.export_options = export_options or {}
	166
	167	def generate_import_record(self, filename, bulkload_state):
	168	"""Generator, yields dicts for nodes found as described in the options.
	169
	170	Args:
	171	filename: Filename to read.
	172	bulkload_state: Passed bulkload_state.
	173
	174	Yields:
	175	Neutral dict, one per row in the CSV file.
	176	"""
	177	self.bulkload_state = bulkload_state
	178	input_stream = open(filename)
	179	input_stream = utf8_recoder(input_stream, self.csv_encoding)
	180
	181	self.dict_generator = csv.DictReader(input_stream, self.column_list,
	182	**self.import_options)
	183	discard_line = self.skip_import_header_row and not self.from_header
	184
	185	for input_dict in self.dict_generator:
	186	if discard_line:
	187	discard_line = False
	188	continue
	189
	190	decoded_dict = {}
	191	for key, value in input_dict.iteritems():
	192	if not self.column_list:
	193	key = unicode(key, 'utf-8')
	194	if value:
	195	value = unicode(value, 'utf-8')
	196	decoded_dict[key] = value
	197	yield decoded_dict
	198
	199	def initialize_export(self, filename, bulkload_state):
	200	"""Initialize the output file.
	201
	202	Args:
	203	filename: Filename to write.
	204	bulkload_state: Passed bulkload_state.
	205	"""
	206	self.bulkload_state = bulkload_state
	207	self.output_stream = open(filename, 'wb')
	208
	209	def __initialize_csv_writer(self, dictionary):
	210	"""Actual initialization, happens on the first entity being written."""
	211	write_header = self.print_export_header_row
	212	if self.from_header:
	213	export_column_list = tuple(dictionary)
	214	else:
	215	export_column_list = self.column_list
	216
	217	self.csv_writer = UnicodeDictWriter(self.output_stream, export_column_list,
	218	self.csv_encoding,
	219	**self.export_options)
	220	if write_header:
	221	self.csv_writer.writerow(dict(zip(export_column_list,
	222	export_column_list)))
	223
	224	def write_dict(self, dictionary):
	225	"""Write one record for the specified entity."""
	226	if not self.csv_writer:
	227	self.__initialize_csv_writer(dictionary)
	228	self.csv_writer.writerow(dictionary)
	229
	230	def finalize_export(self):
	231	self.output_stream.close()
	232
	233

google_appengine_1.3.3/google/appengine/ext/bulkload/simpletext_connector.py [1970-01-01 09:00:00.000000000 +0900]

google_appengine_1.3.4_prerelease/google/appengine/ext/bulkload/simpletext_connector.py [2010-05-13 06:32:48.000000000 +0900]

	1	#!/usr/bin/env python
	2	#
	3	# Copyright 2007 Google Inc.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at

Lodge It

Paste #81

Paste Details