fix(api+cli): improved scope handling; fix CLI

* in APIs, scopes will now be per-method, and if no scope is given,
  we will assume only the API key has to be set. Previously there was
  a wild mix between globally mentioned scopes and method scopes.
* assure CLI generation works so far, for all avaialable APIs

Related to #48

README.md

@@ -30,7 +30,7 @@ Click the image below to see the playlist with all project related content:
 
 Each episode sums up one major step in project development:
 
-* [Episode 1](http://youtu.be/2U3SpepKaBE): How to write 78 APIs in 5s
+* [Episode 1](http://youtu.be/2U3SpepKaBE): How to write 78 APIs in 5 seconds
 
 # Build Instructions
 

src/mako/api/lib/mbuild.mako

@@ -8,7 +8,7 @@
                       indent_by, to_rust_type, rnd_arg_val_for_type, extract_parts, mb_type_params_s,
                       hub_type_params_s, method_media_params, enclose_in, mb_type_bounds, method_response,
                       CALL_BUILDER_MARKERT_TRAIT, pass_through, markdown_rust_block, parts_from_params,
-                      DELEGATE_PROPERTY_NAME, struct_type_bounds_s, supports_scopes, scope_url_to_variant,
+                      DELEGATE_PROPERTY_NAME, struct_type_bounds_s, scope_url_to_variant,
                       re_find_replacements, ADD_PARAM_FN, ADD_PARAM_MEDIA_EXAMPLE, upload_action_fn, METHODS_RESOURCE,
                       method_name_to_variant, unique_type_name, size_to_bytes, method_default_scope,
                       is_repeated_property)
@@ -119,7 +119,7 @@ pub struct ${ThisType}
 % endfor
 ## A generic map for additinal parameters. Sometimes you can set some that are documented online only
     ${api.properties.params}: HashMap<String, String>,
-    % if supports_scopes(auth):
+    % if method_default_scope(m):
 ## We need the scopes sorted, to not unnecessarily query new tokens
     ${api.properties.scopes}: BTreeMap<String, ()>
     % endif
@@ -156,7 +156,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\
         self
     }
 
-    % if supports_scopes(auth):
+    % if method_default_scope(m):
     /// Identifies the authorization scope for the method you are building.
     /// 
     /// Use this method to actively specify which scope should be used, instead the default `Scope` variant
@@ -428,9 +428,7 @@ match result {
     delegate_finish = 'dlg.finished'
     auth_call = 'self.hub.auth.borrow_mut()'
 
-    if supports_scopes(auth):
-        default_scope = method_default_scope(m)
-    # end handle default scope
+    default_scope = method_default_scope(m)
 
     # s = '{foo}' -> ('{foo}', 'foo') -> (find_this, replace_with)
     seen = set()
@@ -573,7 +571,7 @@ else {
         % else:
         let mut url = "${baseUrl}${m.path}".to_string();
         % endif
-        % if not supports_scopes(auth):
+        % if not default_scope:
         <% 
             assert 'key' in parameters, "Expected 'key' parameter if there are no scopes"
         %>
@@ -657,7 +655,7 @@ else {
         % endif
 
         loop {
-            % if supports_scopes(auth):
+            % if default_scope:
             let mut token = ${auth_call}.token(self.${api.properties.scopes}.keys());
             if token.is_none() {
                 token = dlg.token();
@@ -706,7 +704,7 @@ else {
                 let mut client = &mut *self.hub.client.borrow_mut();
                 let mut req = client.borrow_mut().request(${method_name_to_variant(m.httpMethod)}, url.as_ref())
                     .header(UserAgent(self.hub._user_agent.clone()))\
-                    % if supports_scopes(auth):
+                    % if default_scope:
 
                     .header(auth_header.clone())\
                     % endif

src/mako/cli/docs/commands.md.mako

@@ -3,11 +3,13 @@
     from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property)
     from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG,
                      mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode,
-                     FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG)
+                     FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml)
 
     from copy import deepcopy
 
     escape_html = lambda n: n.replace('>', r'\>')
+
+    NO_DESC = 'No description provided.'
 %>\
 <%
     c = new_context(schemas, resources, context.get('methods'))
@@ -18,7 +20,7 @@
     mc = new_method_context(resource, method, c)
 %>\
 ${SPLIT_START} ${subcommand_md_filename(resource, method)}
-% if mc.m.description:
+% if 'description' in mc.m:
 ${mc.m.description}
 % endif # show method description
 % if mc.m.get('scopes'):
@@ -48,26 +50,21 @@ You can set the scope for this method like this: `${util.program_name()} --${SCO
 # Required Scalar ${len(rprops) > 1 and 'Arguments' or 'Argument'}
 % for p in rprops:
 * **<${mangle_subcommand(p.name)}\>**
-    - ${p.get('description') or 'No description provided' | indent_all_but_first_by(2)}
+    - ${p.get('description') or NO_DESC | indent_all_but_first_by(2)}
 % endfor  # each required property (which is not the request value)
 % endif # have required properties
 % if mc.request_value:
+<%
+    request_cli_schema = to_cli_schema(c, mc.request_value)
+%>\
 # Required Request Value
 
 The request value is a data-structure with various fields. Each field may be a simple scalar or another data-structure.
 In the latter case it is advised to set the field-cursor to the data-structure's field to specify values more concisely.
 
 For example, a structure like this:
 ```
-"scalar_int": 5,
-"struct": {
-    "scalar_float": 2.4
-    "sub_struct": {
-        "strings": ["baz", "bar"],
-        "mapping": HashMap,
-    }
-}
-"scalar_str": "foo",
+${cli_schema_to_yaml(request_cli_schema)}
 ```
 
 can be set completely with the following arguments. Note how the cursor position is adjusted the respective fields:
@@ -91,7 +88,7 @@ This method supports the upload of data, using the following protocol${len(mc.me
 
 * **-${UPLOAD_FLAG} ${docopt_mode(protocols)} ${escape_html(FILE_ARG)} ${escape_html(MIME_ARG)}**
 % for mp in mc.media_params:
-    - **${mp.protocol}** - ${mp.description.split('\n')[0]}
+    - **${mp.protocol}** - ${mp.get('description', NO_DESC).split('\n')[0]}
 % endfor # each media param
     - **${escape_html(FILE_ARG)}**
         + Path to file to upload. It must be seekable.
@@ -153,5 +150,5 @@ ${SPLIT_END}
 
 <%def name="_md_property(p)">\
 * **-${PARAM_FLAG} ${mangle_subcommand(p.name)}=${p.type}**
-    - ${p.get('description') or "No description provided" | indent_all_but_first_by(2)}
+    - ${p.get('description') or NO_DESC | indent_all_but_first_by(2)}
 </%def>

src/mako/cli/lib/cli.py

@@ -3,6 +3,7 @@
 import os
 import re
 import collections
+from copy import deepcopy
 
 SPLIT_START = '>>>>>>>'
 SPLIT_END = '<<<<<<<'
@@ -22,12 +23,19 @@
 
 CONFIG_DIR = '~/.google-service-cli'
 
+POD_TYPES = set(('boolean', 'integer', 'number', 'uint32', 'double', 'float', 'int32', 'int64', 'uint64', 'string'))
+
 re_splitters = re.compile(r"%s ([\w\-\.]+)\n(.*?)\n%s" % (SPLIT_START, SPLIT_END), re.MULTILINE|re.DOTALL)
 
 MethodContext = collections.namedtuple('MethodContext', ['m', 'response_schema', 'params', 'request_value', 
                                                          'media_params' ,'required_props', 'optional_props', 
                                                          'part_prop'])
 
+CTYPE_POD = 'pod'
+CTYPE_ARRAY = 'list'
+CTYPE_MAP = 'map'
+SchemaEntry = collections.namedtuple('SchemaEntry', ['container_type', 'actual_property', 'property'])
+
 def new_method_context(resource, method, c):
     m = c.fqan_map[util.to_fqan(c.rtc_map[resource], resource, method)]
     response_schema = util.method_response(c, m)
@@ -46,6 +54,7 @@ def pretty(n):
 def is_request_value_property(mc, p):
     return mc.request_value and mc.request_value.id == p.get(util.TREF)
 
+
 # transform name to be a suitable subcommand
 def mangle_subcommand(name):
     return util.camel_to_under(name).replace('_', '-').replace('.', '-')
@@ -55,12 +64,86 @@ def mangle_subcommand(name):
 def subcommand_md_filename(resource, method):
     return mangle_subcommand(resource) + '_' + mangle_subcommand(method) + '.md'
 
+
 def docopt_mode(protocols):
     mode = '|'.join(protocols)
     if len(protocols) > 1:
         mode = '(%s)' % mode
     return mode
 
+
+# Return schema' with fields dict: { 'field1' : SchemaField(...), 'SubSchema': schema' }
+def to_cli_schema(c, schema):
+    res = deepcopy(schema)
+    fd = dict()
+    res['fields'] = fd
+
+    # util.nested_type_name
+    properties = schema.get('properties', dict())
+    if not properties and 'variant' in schema and 'map' in schema.variant:
+        for e in schema.variant.map:
+            assert util.TREF in e
+            properties[e.type_value] = e
+    # end handle enumerations
+
+    for pn, p in properties.iteritems():
+        def set_nested_schema(ns):
+            if ns.fields:
+                fd[pn] = ns
+        # end utility
+
+        def dup_property():
+            pc = deepcopy(p)
+            if 'type' in pc and pc.type == 'string' and 'Count' in pn:
+                pc.type = 'int64'
+            return pc
+        # end
+
+        if util.TREF in p:
+            if p[util.TREF] != schema.id: # prevent recursion (in case of self-referential schemas)
+                set_nested_schema(to_cli_schema(c, c.schemas[p[util.TREF]]))
+        elif p.type == 'array' and 'items' in p and 'type' in p.get('items') and p.get('items').type in POD_TYPES:
+            pc = dup_property()
+            fd[pn] = SchemaEntry(CTYPE_ARRAY, pc.get('items'), pc)
+        elif p.type == 'object':
+            if util.is_map_prop(p):
+                if 'type' in p.additionalProperties and p.additionalProperties.type in POD_TYPES:
+                    pc = dup_property()
+                    fd[pn] = SchemaEntry(CTYPE_MAP, pc.additionalProperties, pc)
+            else:
+                set_nested_schema(to_cli_schema(c, c.schemas[util.nested_type_name(schema.id, pn)]))
+        elif p.type in POD_TYPES:
+            pc = dup_property()
+            fd[pn] = SchemaEntry(CTYPE_POD, pc, pc)
+        # end handle property type
+    # end
+
+    return res
+
+
+# Convert the given cli-schema (result from to_cli_schema(schema)) to a yaml-like string. It's suitable for 
+# documentation only
+def cli_schema_to_yaml(schema, prefix=''):
+    if not prefix:
+        o = '%s%s:\n' % (prefix, util.unique_type_name(schema.id))
+    else:
+        o = ''
+    prefix += '  '
+    for fn, f in schema.fields.iteritems():
+        o += '%s%s:' % (prefix, mangle_subcommand(fn))
+        if not isinstance(f, SchemaEntry):
+            o += '\n' + cli_schema_to_yaml(f, prefix)
+        else:
+            t = f.actual_property.type
+            if f.container_type == CTYPE_ARRAY:
+                t = '[%s]' % t
+            elif f.container_type == CTYPE_MAP:
+                t = '{ string: %s }' % t
+            o += ' %s\n' % t
+    # end for each field
+    return o
+
+
 # split the result along split segments
 def process_template_result(r, output_file):
     found = False
@@ -74,7 +157,7 @@ def process_template_result(r, output_file):
     for m in re_splitters.finditer(r):
         found = True
         fh = open(os.path.join(dir, m.group(1)), 'wb')
-        fh.write(m.group(2))
+        fh.write(m.group(2).encode('UTF-8'))
         fh.close()
     # end for each match
 

src/mako/lib/util.py

@@ -297,7 +297,7 @@ def mangle_ident(n):
         return n + '_'
     return n
 
-def _is_map_prop(p):
+def is_map_prop(p):
     return 'additionalProperties' in p
 
 def _assure_unique_type_name(schemas, tn):
@@ -342,7 +342,7 @@ def wrap_type(tn):
         if t.type == 'array':
             return wrap_type("%s<%s>" % (rust_type, unique_type_name((nested_type(t)))))
         elif t.type == 'object':
-            if _is_map_prop(t):
+            if is_map_prop(t):
                 return wrap_type("%s<String, %s>" % (rust_type, nested_type(t)))
             else:
                 return wrap_type(nested_type(t))
@@ -725,7 +725,7 @@ def recurse_properties(prefix, rs, s, parent_ids):
                         ns.update((k, deepcopy(v)) for k, v in p.items.iteritems())
 
                     recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs.id))
-                elif _is_map_prop(p):
+                elif is_map_prop(p):
                     recurse_properties(nested_type_name(prefix, pn), rs, 
                                                         p.additionalProperties, append_unique(parent_ids, rs.id))
                 elif 'items' in p:
@@ -838,7 +838,10 @@ def supports_scopes(auth):
     return bool(auth) and bool(auth.oauth2)
 
 # Returns th desired scope for the given method. It will use read-only scopes for read-only methods
+# May be None no scope-based authentication is required
 def method_default_scope(m):
+    if 'scopes' not in m:
+        return None
     default_scope = sorted(m.scopes)[0]
     if m.httpMethod in ('HEAD', 'GET', 'OPTIONS', 'TRACE'):
         for scope in m.scopes: