#@+leo-ver=5-thin
#@+node:ekr.20100119205347.6015: * @file ../doc/leoToDo.txt
#@+all
#@+node:ekr.20101125041212.5845: ** Small improvements
@nocolor-node

To be done early in the release cycle.
#@+node:ekr.20101119030344.5838: *3* script to turn all commands into @g.command nodes
'''Add @g.command(command_name) before all commands.'''

@others

import os ; os.system('cls')

d = c.commandsDict
f_dict = find_all_defs(c)

result1, result2 = [],[]
for f_name in sorted(f_dict): # These are function names, not command name.
    c_name = c.k.inverseCommandsDict.get(f_name) # Get the emacs command name.
    if c_name:
        f = d.get(c_name) # f is the function name that defines a command.
        if f:
            d_name = f.__name__
            s = repr(f)
            tag = '<bound method '
            if s.startswith(tag): s = s[len(tag):]
            i = s.find(' of ')
            if i > -1: s = s[:i]
            aList = [p.h for i,p in f_dict.get(d_name,[])]
            if len(aList) == 1:
                result1.append((d_name,s,aList),)
            else:
                result2.append((d_name,s,aList),)

print('----- duplicate commands -----\n')
for d_name,s,aList in result2:
    print('%s: %s\n%s %s\n' % (d_name,s,len(aList),aList))

print('----- unambiguous commands -----\n')
for d_name,s,aList in result1:
    print('%40s %s' % (d_name,aList[0]))

if 0:
    for name in sorted(d):
        f = d.get(name)
        f_name = f.__name__
        # name is the minibuffer command name, f_name is the function name.
        i,p = find(c,command,f_name)
        adjust(c,f_name,i,p)
#@+node:ekr.20061031131434.3: *4* << about key dicts >>
@nocolor
@
ivars:

c.commandsDict:
    Keys are emacs command names; values are functions f.

k.inverseCommandsDict:
    Keys are f.__name__; values are emacs command names.

k.bindingsDict:
    Keys are shortcuts; values are *lists* of g.bunch(func,name,warningGiven)

k.masterBindingsDict:
    Keys are scope names: 'all','text',etc. or mode names.
    Values are dicts:  keys are strokes, values are g.Bunch(commandName,func,pane,stroke)

k.masterGuiBindingsDict:
    Keys are strokes; value is a list of widgets for which stroke is bound.

k.settingsNameDict:
    Keys are lowercase settings; values are 'real' Tk key specifiers.
    Important: this table has no inverse.

not an ivar (computed by k.computeInverseBindingDict):

inverseBindingDict
    Keys are emacs command names; values are *lists* of shortcuts.
#@+node:ekr.20101119030344.5841: *4* find_all_defs
def find_all_defs (c):

    '''Return a dict containing all function defs.

    Keys are function names, values are lists of (i,p) pairs.'''

    # To do: consider only files that actually generate commands?
    d = {}
    suppress = ('__init__',)
    for p in c.all_unique_positions():
        done,i,s = False,0,p.b
        while not done and i < len(s):
            progress = i
            i = i1 = s.find('def',i)
            if i == -1:
                done = True ; break
            i += 3 # Assures progress.
            if not g.match_word(s,i-3,'def'): continue
            j = g.skip_ws(s,i)
            if j == i: continue
            i = j
            j = g.skip_id(s,i,chars='_')
            if j == i: continue
            name = s[i:j]
            if name not in suppress:
                aList = d.get(name,[])
                aList.append((i1,p.copy()),)
                d[name] = aList
            # g.trace('%30s %s' % (name,p.h))
            i = j
            assert progress < i
    return d
#@+node:ekr.20101119030344.5839: *4* find
def find (c,command,f_name):

    g.trace('%30s %s' % (command,f_name))

    for p in c.all_unique_nodes():
        s = p.b
        i = s.find('def %s' % f_name)
#@+node:ekr.20101119030344.5840: *4* adjust
def adjust(f,i,p):
    pass
#@+node:ekr.20101125041212.5846: *3* Fix bug: ignore f-keys in find/replace patterns
#@+node:ekr.20101124145006.5842: *3* Unify extract commands
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/679614fdd6a76a81#

I think this should be combined with extract-section (ctrl-shift-x), so that

- if the first line of selection is << foo >>, it created a node with
headline << foo >>

- If the  first line is def foo, create a node with headline foo

- Extend as appropriate.
#@+node:ekr.20101124124316.5841: *3* Rename flatten command
@nocolor-node

https://bugs.launchpad.net/leo-editor/+bug/626587

I ask that you please look at this from the newbie perspective that I
have.

If the menu choice said Flatten Selected Outline, or even Flatten
Selected Node, then I would have understood immediately what would
happen on its execution. But the menu choice says neither of these -- it
says Flatten Outline. That gives me a different expectation of what it
will do.
#@+node:ekr.20101123131018.5842: *3* Don't save expansion bits in uA if not saving expansion bits
@nocolor-node

@bool put_expansion_bits_in_leo_files = False

It's illogical to save bits in uA's if they aren't save in in the <v> elements.
#@+node:ekr.20101118113953.5844: *3* Use new reformatParagraph code
@nocolor-node

The following code (untested), changed x(c,...) to c.x(...)

It may be good to put c.findBoundParagraph as a child node.

It should be easy to debug: the only change should be to the top-level
reformatParagraph node.

Unit tests will have to change because the new code moves the cursor to the next
paragraph only if the present paragraph has not actually been changed.
#@+node:ekr.20101118113953.5839: *4* reformatParagraph & helpers
def reformatParagraph (self,event=None,undoType='Reformat Paragraph'):

    """Reformat a text paragraph

    Wraps the concatenated text to present page width setting. Leading tabs are
    sized to present tab width setting. First and second line of original text is
    used to determine leading whitespace in reformatted text. Hanging indentation
    is honored.

    Paragraph is bound by start of body, end of body and blank lines. Paragraph is
    selected by position of current insertion cursor.

"""

    c = self ; body = c.frame.body ; w = body.bodyCtrl

    if g.app.batchMode:
        c.notValidInBatchMode("xxx")
        return

    if body.hasTextSelection():
        i,j = w.getSelectionRange()
        w.setInsertPoint(i)

    oldSel,oldYview,original,pageWidth,tabWidth = c.rp_get_args()
    head,lines,tail = c.findBoundParagraph()
    if lines:
        indents,leading_ws = c.rp_get_leading_ws(lines,tabWidth)
        result = c.rp_wrap_all_lines(indents,leading_ws,lines,pageWidth)
        c.rp_reformat(head,oldSel,oldYview,original,result,tail,undoType)
#@+node:ekr.20101118113953.5840: *5* rp_get_args
def rp_get_args (self):

    '''Compute and return oldSel,oldYview,original,pageWidth,tabWidth.'''

    c = self ; body = c.frame.body ;  w = body.bodyCtrl

    d = c.scanAllDirectives()
    pageWidth = d.get("pagewidth")
    tabWidth  = d.get("tabwidth")

    original = w.getAllText()
    oldSel =  w.getSelectionRange()
    oldYview = body.getYScrollPosition()

    return oldSel,oldYview,original,pageWidth,tabWidth
#@+node:ekr.20101118113953.5841: *5* rp_get_leading_ws
def rp_get_leading_ws (self,lines,tabWidth):

    '''Compute and return indents and leading_ws.'''

    c = self

    indents = [0,0]
    leading_ws = ["",""]

    for i in (0,1):
        if i < len(lines):
            # Use the original, non-optimized leading whitespace.
            leading_ws[i] = ws = g.get_leading_ws(lines[i])
            indents[i] = g.computeWidth(ws,tabWidth)

    indents[1] = max(indents)

    if len(lines) == 1:
        leading_ws[1] = leading_ws[0]

    return indents,leading_ws
#@+node:ekr.20101118113953.5842: *5* rp_reformat
def rp_reformat (self,head,oldSel,oldYview,original,result,tail,undoType):

    '''Reformat the body and update the selection.'''

    c = self ; body = c.frame.body ; w = body.bodyCtrl
    s = w.getAllText()

    # This destroys recoloring.
    junk, ins = body.setSelectionAreas(head,result,tail)

    changed = original != head + result + tail
    if changed:
        # 2010/11/16: stay in the paragraph.
        body.onBodyChanged(undoType,oldSel=oldSel,oldYview=oldYview)
    else:
        # Advance to the next paragraph.
        ins += 1 # Move past the selection.
        while ins < len(s):
            i,j = g.getLine(s,ins)
            line = s[i:j]
            # 2010/11/16: it's annoying, imo, to treat @ lines differently.
            if line.isspace(): ### or line.startswith('@'):
                ins = j+1
            else:
                ins = i ; break

        # setSelectionAreas has destroyed the coloring.
        c.recolor()

    w.setSelectionRange(ins,ins,insert=ins)
    w.see(ins)
#@+node:ekr.20101118113953.5843: *5* rp_wrap_all_lines
def rp_wrap_all_lines (self,indents,leading_ws,lines,pageWidth):

    '''compute the result of wrapping all lines.'''

    trailingNL = lines and lines[-1].endswith('\n')
    lines = [g.choose(z.endswith('\n'),z[:-1],z) for z in lines]

    # Wrap the lines, decreasing the page width by indent.
    result = g.wrap_lines(lines,
        pageWidth-indents[1],
        pageWidth-indents[0])

    # prefix with the leading whitespace, if any
    paddedResult = []
    paddedResult.append(leading_ws[0] + result[0])
    for line in result[1:]:
        paddedResult.append(leading_ws[1] + line)

    # Convert the result to a string.
    result = '\n'.join(paddedResult)
    if trailingNL: result = result + '\n'

    return result
#@+node:ekr.20101116104701.5828: *3* Make promote-child-body an official command
#@+node:ekr.20101116100300.5827: *3* Write script file to temp file if @bool write_script_file is False
@nocolor-node

It's important that g.pdb get the sources for *all* Leo scripts.

Postings:

When @bool write_script_file is True the execute-script file uses
execfile (or its Python 3k equivalent) to execute scripts.

This allows g.pdb (or pudb) to know the text of Leo scripts!

I want to emphasize here how important this change is.  After 4.8 final I'll add code to use a temp file to hold sources if 
@bool write_script_file is False.

In the meantime, I strongly urge all script writers to set @bool write_script_file = True.

Depending on your installation, especially on *nix, the default location for the file may be a problem.  In that case, set
@string script_file_path to some convenient place, say in your home directory.
#@+node:ekr.20101104191857.5820: *3* Idea: presentation tool
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/4ea2d3f7d2c68106#

Ville:

Create one QWebView window, zoom it in to have large fonts.

Create @button that converts current node containing restructuredtext to html,
and pushes that html to QWebView.

Voila', instant presentation tool. The webview window would be on projector, and
leo would be in your private computer. You can easily edit the text, or find new
interesting slides to present in privacy of your own screen.

#@+node:ekr.20101024062147.6014: ** Autocompleter
# This can be done in the trunk.
#@+node:ekr.20090131200406.14: *3* Auktocompleter notes
#@+node:ekr.20080113165010: *4* About auto completion
@nocolor

Summary: inspect when possible, ask for types otherwise.

    os.? gives a list of all instances of os module.
    x.? gives a list of all known classes.
    x.# gives list of all known functions.
    self.? gives all known methods and ivars of the enclosing class.
    The user will be able to specify type conventsions, like Leo's c,g,p,t,v vars.

Completed (mostly): user interface stuff.

Performance isn't too hard:
    - Do all scanning in separate threads.
    - Associate node info with tnodes.
    - Update node info when deselecting a node (in tree base class)

Parsing:
    - Forgiving parser is essentially complete.
    - It's easy to parse python def lines.
#@+node:ekr.20081005065934.12: *4* Links from Ville re Scintilla
@nocolor-node

It seems Scintilla relies on externally generated api description
files to provide autocompletion. Links that may be of interest:

http://www.riverbankcomputing.co.uk/static/Docs/QScintilla2/classQsciAPIs.html#b0f824492bb0f3ca54edb4d46945a3de

http://www.burgaud.com/scite-java-api/

http://scintilla.sourceforge.net/tags2api.py

http://www.koders.com/python/fid7000B9C96CF2C6FB5BCE9DF700365C5B2A1F36A7.aspx?s=gtk#L53
#@+node:ekr.20060927173836.1: *4* Make calltips and autocompleter 'stateless'
@nocolor

Disabled these binding:

auto-complete-force = None # This command needs work before it is useful. Ctrl-period
show-calltips-force = None # This command needs work before it is useful. Alt-parenleft

The problem is that autocompletion depends on state: self.leadinWord,
prevObjects, etc. Thus, it's not presently possible to start the proces
anywhere. Similar remarks apply to calltips, which relies on autocompleter
state.

This is a complex problem, and not very serious now that there is an easy way of
toggling autocompleter and calltips on and off.

@color
#@+node:ekr.20080110082845: *4* pyxides: code completion
@nocolor

Python code completion module


From: "Tal Einat" <talei...@gmail.com>
Date: Wed, 6 Jun 2007 20:57:18 +0300

I've been developing IDLE over the past 2 years or so. Even before
that, I helped a friend of mine, Noam Raphael, write IDLE's
auto-completion, which is included in recent versions of IDLE.

Noam wrote the original completion code from scratch, and AFAIK every
Python IDE which features code completion has done the same. Surely
there is -some- functionality which could be useful cross-IDE?
Retrieving possible completions from the namespace, for example. And
we should be learning from each-others' ideas and experiences.

So how about we design a generic Python completion module, that
each IDE could extend, and use for the completion logic?



From: "Ali Afshar" <aafs...@gmail.com>
Date: Wed, 6 Jun 2007 19:06:01 +0100

I am very keen for this. I will help where it is required. PIDA
currently has no code completion (outside what vim/emacs provide),



From: "phil jones" <inters...@gmail.com>
Date: Wed, 6 Jun 2007 11:07:33 -0700

What functions would we ask for a code completion module?

Presumably recognition of the beginnings of
- a) python keywords
- b) classes and functions defined earlier in this file?
- c) in scope variables?

As python is dynamically typed, I guess we can't expect to know the
names of methods of objects?



From: "Ali Afshar" <aafs...@gmail.com>
Date: Wed, 6 Jun 2007 19:13:10 +0100

> Presumably recognition of the beginnings of
> - a) python keywords
> - b) classes and functions defined earlier in this file?
> - c) in scope variables?

does c) include: d) imported modules



From: Nicolas Chauvat <nicolas.chau...@logilab.fr>
Date: Wed, 6 Jun 2007 20:17:30 +0200

> >Presumably recognition of the beginnings of
> >- a) python keywords
> >- b) classes and functions defined earlier in this file?
> >- c) in scope variables?

> does c) include: d) imported modules

For code-completion, I suppose astng[1] could be useful.

1: http://www.logilab.org/project/eid/856



From: Stani's Python Editor <spe.stani...@gmail.com>
Date: Wed, 06 Jun 2007 20:48:41 +0200

A good point. I think we all have been thinking about this. Important
issues for the design is the extraction method and the sources.

*the method*
Importing is a lazy, but accurate way of importing, but is security wise
not such a good idea. Parsing throught an AST compiler is better,
however more difficult. Here are two options.

From version 2.5 the standard Python compiler converts internally the
source code to an abstract syntax tree (AST) before producing the
bytecode. So probably that is a good way to go as every python
distribution has this battery included.

As Nicolas suggested earlier on this mailing list, there is another
option: the AST compiler in python or PyPy:

On Mar 14 2006, 12:16 am, Nicolas Chauvat <nicolas.chau...@logilab.fr>
wrote:

> > WingIDE use anASTgenerator written in C (but cross-platform),
> > lightningly quick, and open sourced. This could be a potential
> > starting point.

> > Additionally isn't Python2.5 planned to have a C-written compiler?

> PyPy also produced an improved parser/compiler.

> http://codespeak.net/pypy/dist/pypy/doc/index.html
> http://codespeak.net/pypy/dist/pypy/module/recparser/

But if it could be done with the standard one it is one dependency less.

*the sources*
In the design we could define first the sources:
1 external imported modules from the pythonpath
2 local modules relative to the current file or context dependent
(Blender, Gimp, ...)
3 inner code

For 1:
It might be a good idea to have a function which scans all the modules
from the pythonpath or one specific module to cache all autocompletion
and calltip information of all classes, methods and doc strings. Why?
Modules in the pythonpath don't change so often. With some criteria
(file name, time stamp, size, ...) you could check if updates are
necessary at startup. Having a readymade 'database' (could be python
dictionary or sqlite database) for autocompletion/call tips would speed
up things (and is also more secure if you are importing rather than
parsing. For example trying to provide gtk autocompletion in a wxPython
by importing is problematic).

For 2:
Here you load the parser on demand. Autocompletion/calltip information
can be added to the database.

For 3:
A different kind of parser needs to be used here as per definition code
you edit contains errors while typing. External modules are retrieved
from 1 and 2, for internal code you can scan all the words and add them
to the autocomplete database. As a refinement you can give special
attention to 'self'. Also for calltips you can inherit when there are
assignments, eg
frame = Frame()
than frame inherits autocomplete & calltip information from Frame.

So autocompletion & calltips deals with two steps: extraction and
'database'. If someone has a good parser already, we could use it.
Otherwise we can define an API for the extraction and maybe lazily
implement it first with importing and concentrate first on the
'database'. When the database is ready we can implement the parsing. You
could also implement the parsing first, but than it takes longer before
you have results. Of course the library is GUI independent, it only
works with strings or lists.

What concerns SPE, it uses importing for autocompletion (1+2) and does
internal code analysis for local code (however without the inheriting).

Tal, how does IDLE's autocompletion works?

Stani



From: Stani's Python Editor <spe.stani...@gmail.com>
Date: Wed, 06 Jun 2007 20:53:10 +0200

Nicolas Chauvat wrote:
> On Wed, Jun 06, 2007 at 07:13:10PM +0100, Ali Afshar wrote:
>>> Presumably recognition of the beginnings of
>>> - a) python keywords
>>> - b) classes and functions defined earlier in this file?
>>> - c) in scope variables?
>> does c) include: d) imported modules

> For code-completion, I suppose astng[1] could be useful.

> 1: http://www.logilab.org/project/eid/856

How dependent/independent is this from the standard AST compiler or
PyPy? Is it more IDE friendly? Is it based on it or a total independent
implementation?



From: "Ali Afshar" <aafs...@gmail.com>
Date: Wed, 6 Jun 2007 19:59:13 +0100

> A good point. I think we all have been thinking about this. Important
> issues for the design is the extraction method and the sources.

> *the method*
> Importing is a lazy, but accurate way of importing, but is security wise
> not such a good idea. Parsing throught an AST compiler is better,
> however more difficult. Here are two options.

> From version 2.5 the standard Python compiler converts internally the
> source code to an abstract syntax tree (AST) before producing the
> bytecode. So probably that is a good way to go as every python
> distribution has this battery included.

> As Nicolas suggested earlier on this mailing list, there is another
> option: the AST compiler in python or PyPy:

What concerns me about these is whether they would work in a module
which has a syntax error.

I believe Wing's compiler bit of their code completion is open source.
I remember having seen the code.



From: Stani <spe.stani...@gmail.com>
Date: Wed, 06 Jun 2007 12:08:00 -0700

> What concerns me about these is whether they would work in a module
> which has a syntax error.

> I believe Wing's compiler bit of their code completion is open source.
> I remember having seen the code.

It is indeed, but is implemented in C, which means an extra dependency
and not a 100% python solution. Normally modules (especially in the
pythonpath) which you import don't have syntax errors. Maybe logilabs
implementation handles syntax errors well as it is developed for
PyLint. Nicolas?



From: "Tal Einat" <talei...@gmail.com>
Date: Wed, 6 Jun 2007 22:34:41 +0300

> As python is dynamically typed, I guess we can't expect to know the
> names of methods of objects?

Well, the dir() builtin does just that, though there can be attributes
which won't be included therein. However, the builtin dir() can be
overridden... and ignoring it can break libraries like RPyC which
define a custom dir() function just for this purpose.

This issue has already been run in to by RPyC (an Python RPC lib). The
main developr went ahead and suggested adding a __dir__ method which
will return a list of attributes, and IIRC he has already implemented
a patch for this, and it will likely enter Python2.6.

Until then, I guess we're going to have to rely on dir for this.



From: "Josiah Carlson" <josiah.carl...@gmail.com>
Date: Wed, 6 Jun 2007 12:42:01 -0700

For reference, PyPE auto-parses source code in the background, generating
(among other things) a function/class/method hierarchy.  Its autocomplete
generally sticks to global functions and keywords, but when doing
self.method lookups, it checks the current source code line, looks up in its
index of classes/methods, and trims the results based on known methods in
the current class in the current source file.

It certainly isn't complete (it should try to check base classes of the
class in the same file, it could certainly pay attention to names assigned
in the current scope, the global scope, imports, types of objects as per
WingIDE's assert isinstance(obj, type), etc.), but it also makes the
computation fairly straightforward, fast, and only in reference to the
current document.



From: "Tal Einat" <talei...@gmail.com>
Date: Wed, 6 Jun 2007 22:52:08 +0300

> Tal, how does IDLE's autocompletion works?

Much like Stani said, since Python is interpreted, collection of
possible completions splits into two methods:
1) source code analysis
2) dynamic introspection

Of course, we could do either or a combination of both.

IDLE just uses introspection: since IDLE always has a python shell
running, it just completes according to the shell's state (plus
built-in keywords and modules). This is a very simple method,
obviously lacking. It does allow the user some control of the
completion, though - just import whatever you want to be completable
in the shell. However, introspection is all that is needed in a Python
shell, which is the major reason this is the method used in IDLE.



From: Nicolas Chauvat <nicolas.chau...@logilab.fr>
Date: Wed, 6 Jun 2007 23:59:32 +0200


> How dependent/independent is this from the standard AST compiler or
> PyPy? Is it more IDE friendly? Is it based on it or a total independent
> implementation?

It is independent from PyPy.

The above web page says:

"""
Python Abstract Syntax Tree New Generation

The aim of this module is to provide a common base representation of
python source code for projects such as pychecker, pyreverse,
pylint... Well, actually the development of this library is essentialy
governed by pylint's needs.

It extends class defined in the compiler.ast [1] module with some
additional methods and attributes. Instance attributes are added by a
builder object, which can either generate extended ast (let's call
them astng ;) by visiting an existant ast tree or by inspecting living
object. Methods are added by monkey patching ast classes.Python
Abstract Syntax Tree New Generation

The aim of this module is to provide a common base representation of
python source code for projects such as pychecker, pyreverse,
pylint... Well, actually the development of this library is essentialy
governed by pylint's needs.

It extends class defined in the compiler.ast [1] module with some
additional methods and attributes. Instance attributes are added by a
builder object, which can either generate extended ast (let's call
them astng ;) by visiting an existant ast tree or by inspecting living
object. Methods are added by monkey patching ast classes.
"""

From: "Sylvain Thénault" <thena...@gmail.com>
Date: Wed, 13 Jun 2007 10:51:04 +0200

> Please let me involve Sylvain in the discussion. As the main author of
> pylint and astng, he will provide better answers.

well logilab-astng is basically a big monkey patching of the compiler
package from the stdlib, so you can't get an astng representation from a
module with syntax errors in. However inference and most others
navigation methods (which are basically the value added by astng) are
"syntax error resilient" : if a dependency module (direct or indirect)
contains a syntax error, you don't get any exception, though since some
information is missing you can miss some results you'ld get if the
faulting module were parseable.



From: "Tal Einat" <talei...@gmail.com>
Date: Tue, 31 Jul 2007 10:33:33 +0300

Since astng already does some inference (which we definitely want!)
and is based on the standard Python AST compiler, it sounds like our
#1 candidate. I think we should give the code a serious once-over and
see how well it fits our requirements, and if it can be adapted to
better handle errors. Any volunteers?

Also, has anyone used astng for completion, calltips, or something
similar? Or the standard AST compiler, for that matter?



From: "Tal Einat" <talei...@gmail.com>
Date: Tue, 31 Jul 2007 10:40:11 +0300

How does PyPE parse code? Home-rolled, standard AST compiler, something else?

It seems to me we should try to come up with an algorithm for parsing,
before getting to the code. All of the details you mentioned -
noticing assignments, using base-class methods, etc. - could be better
defined and organized this way. Perhaps we could brainstorm on this in
a wiki?



From: "Tal Einat" <talei...@gmail.com>
Date: Tue, 31 Jul 2007 11:38:40 +0300

Sorry for being away for such a long time. I hope we can get this
conversation rolling again, and get started with the actual work.

I'll try to sum up what has been said so far, and how I see things.

== Top Priorities ==
* Can we implement a parser based on the standard Python AST compiler
(or astng)? For example, can syntax errors be handled well?
* Is importing reasonable security-wise? If not, can it be made secure?

== General issues ==
* Do we aim for just completion, or also calltips? Perhaps also other
meta-data, e.g. place defined, source code, ... (see IPython's '??')
* Dependencies - do we want to allow C-extensions, or are we going for
a Python-only solution? (IDLE would only use such a Python-only tool.)
It seems that we want to pre-process most of the data in the
background, so I don't see why we would want to do this in C for
efficiency reasons.

== Completion sources ==
1) Importing "external" modules
2) Importing/Parsing "local" modules
3) Parsing the current file
4) Using objects/modules from the shell (e.g. IDLE has both editor
windows and a Python shell)

== Importing ==
* Stani mentioned that importing is problematic from a security point
of view. What are the security issues? Are they really an issue for an
IDE? If so, perhaps we could overcome this by importing in some kind
of "sandbox"?
* What are the pros and cons of Importing vs. Parsing?
* If importing is always preferable to parsing unless there's a syntax
error, perhaps try to import and parse on failure?

== Parsing ==
* This is going to be the most complex method - I think we should have
a general idea of how this should work before starting an
implementation. I suggest hashing ideas out on a wiki, since there a
lot of details to consider.
* Can a parser based on the standard AST compiler (or astng) work? Is
there a way to deal with errors? (HIGH PRIORITY!)
* There are other existing, open-source implementations out there -
WingIDE, PyPE have been mentioned. Any others? We should collect these
so we can use the code for learning, and perhaps direct use (if
possible license-wise).

== Shell ==
This is relatively straight-forward - just use dir(). This should be
optional, for use by IDEs which have a shell (support multiple
shells?).

Some known issues from IDLE and PyCrust:
* Handle object proxies such as RPC proxies (e.g. RPyC)
* Handle ZODB "ghost" objects
* Watch out for circular references
* Watch out for objects with special __getattr__/__hasattr__
implementations (for example xmlrpc, soap)

== Persistence ==
* Stani mentioned a 'database'. I feel Sqlite should be at most
optional, to reduce dependencies.
* Do we really want to have the data persistent (between IDE
sessiosns)? If so, we need to support simultaneous instances of the
IDE so they don't corrupt the data. Any other issues? (I have a
feeling this would better be left for later stages of development.)



From: "Tal Einat" <talei...@gmail.com>
Date: Tue, 31 Jul 2007 12:22:59 +0300

One more note: We should distinguish between completion in an editor
and completion in a shell. The conversation up until now has focused
on editors, which is reasonable since that is the problematic scene. I
think a generic Python completion library should support completion in
both contexts, especially if it uses can use a shell's namespace for
completion in the editor.



From: "Ali Afshar" <aafs...@gmail.com>
Date: Tue, 31 Jul 2007 11:20:19 +0100

I have just implemented a completion mockup using Rope (which is a
refactoring library). It works quite nicely, and definitely worth a
look.

http://rope.sourceforge.net/

It even achieves this kind of completion:

class Banana(object):
    def do_something(self):
         return

def foo():
    return [Banana(), Banana()]

foo()[0].<complete> includes do_something

Which seems pretty impressive to me.



From: "Tal Einat" <talei...@gmail.com>
Date: Tue, 31 Jul 2007 20:12:50 +0300

Wow, Rope does look very impressive! A quick look at the code tells me
that a lot of work has been invested in it.

So we have one existing Python-only solution. We should evaluate it -
see what it can and can't do, and perhaps take a look at the overall
design.

I'm CC-ing Rope's developer, Ali. Hopefully Ali can help us quickly
understand Rope's code analysis capabilities.

Ali, could you elaborate a bit on what kinds of completion Rope can
do, and the methods it uses? We would especially like to know how your
static and dynamic inference work, what they can accomplish, and what
their limitations are.



From: "Ali Afshar" <aafs...@gmail.com>
Date: Tue, 31 Jul 2007 19:45:15 +0100

> Ali, could you elaborate a bit on what kinds of completion Rope can
> do, and the methods it uses? We would especially like to know how your
> static and dynamic inference work, what they can accomplish, and what
> their limitations are.

Well, I haven't really looked at the code. But I can tell you this:

from rope.ide.codeassist import PythonCodeAssist
from rope.base.project import Project
for compl in PythonCodeAssist(Project(package_root)).assist(buffer,
offset).completions:
    print compl

And that is as far as I really got. I expect to get a better look at
it later in the week though...


From: "Josiah Carlson" <josiah.carl...@gmail.com>
Date: Wed, 1 Aug 2007 00:26:14 -0700

> How does PyPE parse code? Home-rolled, standard AST compiler, something else?

The compiler for syntactically correct Python, a line-based compiler
for broken Python.  TO generate a method list for self.methods, using
the current line number, I discover the enclosing class, check the
listing of methods for that class (generated by the compiler or
line-based parsers), and return a valid list for the specified prefix.
 It doesn't walk the inheritance tree, it doesn't do imports, etc.

> It seems to me we should try to come up with an algorithm for parsing,
> before getting to the code. All of the details you mentioned -
> noticing assignments, using base-class methods, etc. - could be better
> defined and organized this way. Perhaps we could brainstorm on this in
> a wiki?

A wiki would be fine, the one for this mailing list would likely be
best (if it is still up and working).  Then again, Rope looks quite
nifty.  I may have to borrow some of that source ;)


Discussion subject changed to "Fwd: Python code completion module" by Tal Einat

From: Ali Gholami Rudi <aligr...@gmail.com>
Date: Aug 1, 2007 5:50 PM

First of all I should note that rope's main goal was being a
refactoring tool and a refactoring tool needs to know a lot about
python modules.  `rope.base` package provides information about python
modules.

Actually what ropeide provides as auto-completion is defined in
`rope.ide.codeassist` module.  This module almost does nothing but use
`rope.base`.  Since `rope.ide` package is not included in the rope
library (which has been separated from ropeide since 0.6m4) it lacks
good documentation and the API might not be easy to use (most of it is
written in the first months of rope's birth).

> ..., could you elaborate a bit on what kinds of completion Rope can
> do, ...

I don't know what to say here.  Well, actually it tries to use the
source code as much as possible and infer things from it.  So I can
say that it can complete any obvious thing that can be inferred by a
human.  Like this is the first parameter of a method and after dots
its attributes can appear or these modules are imported so their names
and contents are available or this is an instance of some known type
and we know its attributes and ... .  Try ropeide (it uses emacs-like
keybinding, C-/ for completion; see ~/.rope if you want to change
that); it completes common cases (and sometimes completes things you
don't expect it to!).

> ..., and the methods it uses?

Rope analyzes python source code and AST.  Rope used to use the
`compiler` module till 0.5 and now it uses `_ast` module.

> We would especially like to know how your
> static and dynamic inference work, what they can accomplish

There are a few examples in docs/overview.txt.  Unit-test modules like
`ropetest.base.objectinfertest` and `advanced_oi_test` might help,
too.  Also have a look at `rope.base.oi.__init__` pydoc for an
overview of how they work; (I'm afraid it is a bit out of date and
carelessly written.)  The idea behind rope's object inference is to
guess what references (names in source-code) hold.  They collect
information about code when they can and use them later.

>..., and what their limitations are.

Many things in rope are approximations that might be exact if some
conditions hold.  For instance rope might assume that every normal
reference in module scope holds only one kind of object.  Apart from
these assumptions both SOI and DOI have their own disadvantages; For
instance SOI fails when dynamic code is evaluated while DOI does not.
Or DOI is slower than SOI.  (Well, after recent enhancements to rope's
SOI I rarely use DOI).

I tried to answer as short as possible.  If there are questions on
specific parts of rope, I'll be happy to answer.

By the way, I tried to reply this mail to the group, but it seems that
your group requires subscription for posting, so I've sent it to you,
instead.
#@+node:ekr.20071106083149: *4* Recent post
@killcolor

In general, autocompletion is a tricky problem. Consider:

- There may be no 'clean' version of the source code that you want to
auto-complete: you may be creating a new node, or a new file, and the source
code, being incomplete, will not parse correctly.

- Except in special circumstances, there is no 'real' object corresponding to s,
so there is no way to use Python's inspect module on s. Modules are an
exception: the autocompleter can handle existing modules fairly well. Try "os."
or "os.path." for example.


It might be possible to generalize c.k.defineObjectDict so that the user
could specify autocompleter conventions, say in an @autocompleter node in an
@settings tree.
#@+node:ekr.20100506062734.5756: *4* Ville post re autocompleter
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/ee84e52fba476950

> Some of the UI's I've worked with involved going into a state where
> you picked from a numbered, or lettered list of possibilities
> generated based on the currently typed "trigger".  e.g. and expansion
> of trigger "ct" puts a ine like the following in, say, the status
> line:

Note that all design work is unnecessary - this is basically a simple
matter of bugfixing.

qtGui has this method:

http://pastebin.com/GmhZJqDd

Once you can call

w.showCompleter(['hello', 'helloworld'], mycallback )

and select the correct alternative naturally with keyboard (now you
need mouse), almost all of the work is done.

The rest can be provided by codewisecompleter, or any other kind of
completer we may wish to add (e.g. hippie-expand, abbrev expand...).

All discussion regarding the ui design only helps to sidetrack us from
getting a completer that behaves the way completers are "supposed to
behave" on 2010, i.e. the way provided by QCompleter.

-- 
Ville M. Vainio 
#@+node:ekr.20101004092958.6008: *3* codewise stuff
#@+node:ekr.20101004092958.5964: *4* codewise crash

c:\leo.repo\trunk>python -m codewise setup

c:\leo.repo\trunk>c:\Python31\python.exe -m codewise setup
Creating template c:\Users\edreamleo/.ctags
Initializing CodeWise db at c:\Users\edreamleo/.codewise.db
c:\Users\edreamleo/.codewise.db
<sqlite3.Connection object at 0x02B881A0>

c:\leo.repo\trunk>python -m codewise init

c:\leo.repo\trunk>c:\Python31\python.exe -m codewise init
Initializing CodeWise db at c:\Users\edreamleo/.codewise.db
c:\Users\edreamleo/.codewise.db
<sqlite3.Connection object at 0x02BA21A0>

c:\leo.repo\trunk>python -m codewise parse c:\leo.repo\trunk

c:\leo.repo\trunk>c:\Python31\python.exe -m codewise parse
Traceback (most recent call last):
  File "c:\Python31\lib\runpy.py", line 128, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "c:\Python31\lib\runpy.py", line 34, in _run_code
    exec(code, run_globals)
  File "c:\Python31\lib\site-packages\codewise.py", line 454, in <module>
    main()
  File "c:\Python31\lib\site-packages\codewise.py", line 440, in main
    cmd_parse(args)
  File "c:\Python31\lib\site-packages\codewise.py", line 368, in cmd_parse
    assert args
AssertionError

c:\leo.repo\trunk>scripts

c:\leo.repo\trunk>cd c:\Scripts

c:\Scripts>ed python.bat

c:\Scripts>echo off
Could Not Find c:\Scripts\*.~?~

c:\Scripts>tr

c:\Scripts>cd c:\leo.repo\trunk

c:\leo.repo\trunk>python -m codewise parse c:\leo.repo\trunk

c:\leo.repo\trunk>c:\Python31\python.exe -m codewise parse c:\leo.repo\trunk
c:\Users\edreamleo/.codewise.db
ctags -R --sort=no -f - c:\leo.repo\trunk

c:\leo.repo\trunk>python -m codewise parse c:\python31\lib

c:\leo.repo\trunk>c:\Python31\python.exe -m codewise parse c:\python31\lib
c:\Users\edreamleo/.codewise.db
ctags -R --sort=no -f - c:\python31\lib
Traceback (most recent call last):
  File "c:\Python31\lib\runpy.py", line 128, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "c:\Python31\lib\runpy.py", line 34, in _run_code
    exec(code, run_globals)
  File "c:\Python31\lib\site-packages\codewise.py", line 454, in <module>
    main()
  File "c:\Python31\lib\site-packages\codewise.py", line 440, in main
    cmd_parse(args)
  File "c:\Python31\lib\site-packages\codewise.py", line 370, in cmd_parse
    cw.parse(args)
  File "c:\Python31\lib\site-packages\codewise.py", line 307, in parse
    self.feed_ctags(f)
  File "c:\Python31\lib\site-packages\codewise.py", line 250, in feed_ctags
    fil = fields[1]
IndexError: list index out of range

c:\leo.repo\trunk>
#@+node:ekr.20101004092958.6009: *4* codewise.py
@first #!/usr/bin/env python

<< docstring >>
<< usage >>
<< decls >>

@language python
@tabwidth -4

@others

if __name__ == "__main__":
    main()
#@+node:ekr.20101004092958.6010: *5* << docstring >>
""" CodeWise - global code intelligence database

Why codewise:

- Exuberant ctags is an excellent code scanner
- Unfortunately, TAGS file lookup sucks for "find methods of this class"
- TAGS files can be all around the hard drive. CodeWise database is
  just one file (by default ~/.codewise.db)
- I wanted to implement modern code completion for Leo editor

- This is usable as a python module, or a command line tool.

"""
#@+node:ekr.20101004092958.6011: *5* << usage >>
USAGE = """
codewise setup
 (Optional - run this first to create template ~/.ctags)

codewise init
 Create/recreate the global database

codewise parse /my/project /other/project
 Parse specified directories (with recursion) and add results to db

codewise m
 List all classes

codewise m MyClass
 Show all methods in MyClass

codewise f PREFIX
 Show all symbols (also nonmember functiosn) starting with PREFIX.
 PREFIX can be omitted to get a list of all symbols

codewise parseall
 Clear database, reparse all paths previously added by 'codewise parse'

codewise sciapi pyqt.api
 Parse an api file (as supported by scintilla, eric4...)

Commands you don't probably need:

codewise tags TAGS
 Dump already-created tagfile TAGS to database

""" 
#@+node:ekr.20101004092958.6012: *5* << decls >>
import os,sys
import sqlite3

isPython3 = sys.version_info >= (3,0,0) # EKR.

DEFAULT_DB = os.path.expanduser("~/.codewise.db")

DB_SCHEMA = """
BEGIN TRANSACTION;
CREATE TABLE class (id INTEGER PRIMARY KEY, file INTEGER,  name TEXT, searchpattern TEXT);
CREATE TABLE file (id INTEGER PRIMARY KEY, path TEXT);
CREATE TABLE function (id INTEGER PRIMARY KEY, class INTEGER, file INTEGER, name TEXT, searchpattern TEXT);
CREATE TABLE datasource (type TEXT, src TEXT);

CREATE INDEX idx_class_name ON class(name ASC);
CREATE INDEX idx_function_class ON function(class ASC);

COMMIT;
"""

#@+node:ekr.20101004092958.6013: *5* Top level
#@+node:ekr.20101004092958.6014: *6* Commands
#@+node:ekr.20101004092958.6015: *7* cmd_functions
def cmd_functions(args):
    cw = CodeWise()

    if args:
        funcs = cw.get_functions(args[0])
    else:
        funcs = cw.get_functions()
    lines = list(set(el[0] + "\t" + el[1] for el in funcs))
    lines.sort()
    printlines(lines)
    return lines # EKR
#@+node:ekr.20101004092958.6016: *7* cmd_init
def cmd_init(args):

    print("Initializing CodeWise db at", DEFAULT_DB)
    if os.path.isfile(DEFAULT_DB):
        os.remove(DEFAULT_DB)

    cw = CodeWise()
#@+node:ekr.20101004092958.6017: *7* cmd_members
def cmd_members(args):
    cw = CodeWise()
    if not args:
        lines = cw.classcache.keys()
        lines.sort()
        printlines(lines)        
        return [] # EKR

    mems = cw.get_members([args[0]])
    lines = list(set(el + "\t" + pat for el, pat in mems))
    lines.sort()
    printlines(lines)
    return lines # EKR
#@+node:ekr.20101004092958.6018: *7* cmd_parse
def cmd_parse(args):
    assert args
    cw = CodeWise()
    cw.parse(args)
#@+node:ekr.20101004092958.6019: *7* cmd_parseall
def cmd_parseall(args):
    cw = CodeWise()
    cw.parseall()
#@+node:ekr.20101004092958.6020: *7* cmd_scintilla
def cmd_scintilla(args):
    cw = CodeWise()
    for fil in args:
        f = open(fil)
        cw.feed_scintilla(f)
        f.close()
#@+node:ekr.20101004092958.6021: *7* cmd_setup
def cmd_setup(args):

    ctagsfile = os.path.expanduser("~/.ctags")
    print("Creating template",ctagsfile)
    assert not os.path.isfile(ctagsfile)
    open(ctagsfile, "w").write("--exclude=*.html\n--exclude=*.css\n")
    cmd_init(args)
#@+node:ekr.20101004092958.6022: *7* cmd_tags
def cmd_tags(args):
    cw = CodeWise()
    cw.feed_ctags(open(args[0]))
#@+node:ekr.20101004092958.6023: *6* main
def main():
    if len(sys.argv) < 2:
        print(USAGE)
        return
    cmd = sys.argv[1]
    #print "cmd",cmd
    args = sys.argv[2:]
    if cmd == 'tags':
        cmd_tags(args)
    elif cmd == 'm':
        cmd_members(args)
    elif cmd == 'f':
        cmd_functions(args)        
    elif cmd =='parse':
        cmd_parse(args)
    elif cmd =='parseall':
        cmd_parseall(args)
    elif cmd =='sciapi':
        cmd_scintilla(args)
    elif cmd == 'init':
        cmd_init(args)
    elif cmd == 'setup':
        cmd_setup(args)
#@+node:ekr.20101004092958.6024: *6* printlines
def printlines(lines):
    for l in lines:
        try:
            print(l)
        except Exception: # EKR: UnicodeEncodeError:            
            pass
#@+node:ekr.20101004092958.6025: *6* run_ctags
def run_ctags(paths):
    cm = 'ctags -R --sort=no -f - ' + " ".join(paths)
    print(cm)
    f = os.popen(cm)
    return f
#@+node:ekr.20101004092958.6026: *5* class CodeWise
class CodeWise:
    @others
#@+node:ekr.20101004092958.6027: *6* __init__
def __init__(self, dbpath = None):

    if dbpath is None:
        # use "current" db from env var
        dbpath = DEFAULT_DB

    print(dbpath)

    self.reset_caches()

    if not os.path.exists(dbpath):
        self.createdb(dbpath)
    else:
        self.dbconn = c = sqlite3.connect(dbpath)
        self.create_caches()
#@+node:ekr.20101004092958.6028: *6* createdb
def createdb(self, dbpath):

    self.dbconn = c = sqlite3.connect(dbpath)
    print(self.dbconn)
    c.executescript(DB_SCHEMA)
    c.commit()
    c.close()
#@+node:ekr.20101004092958.6029: *6* create_caches
def create_caches(self):
    """ read existing db and create caches """

    c = self.cursor()

    c.execute('select id, name from class')
    for idd, name in c:
        self.classcache[name] = idd

    c.execute('select id, path from file')
    for idd, name in c:
        self.filecache[name] = idd

    c.close()
    #print self.classcache
#@+node:ekr.20101004092958.6030: *6* reset_caches
def reset_caches(self):
    self.classcache = {}
    self.filecache = {}

    self.fileids_scanned = set()
#@+node:ekr.20101004092958.6031: *6* cursor
def cursor(self):
    return self.dbconn.cursor()
#@+node:ekr.20101004092958.6032: *6* class_id
def class_id(self, classname):
    """ return class id. May create new class """

    if classname is None:
        return 0

    idd = self.classcache.get(classname)
    if idd is None:
        c = self.cursor()
        c.execute('insert into class(name) values (?)' , [classname])
        c.close()
        idd = c.lastrowid
        self.classcache[classname] = idd
    return idd
#@+node:ekr.20101004092958.6033: *6* get_members
def get_members(self, classnames):

    clset = set(classnames)

    class_by_id = dict((v,k) for k,v in self.classcache.items())
    file_by_id = dict((v,k) for k,v in self.filecache.items())


    res = []
    for name, idd in self.classcache.items():
        if name in clset:
            c = self.cursor()
            #print idd
            c.execute('select name, class, file, searchpattern from function where class = (?)',(idd,))

            for name, klassid, fileid, pat in c:
                res.append((name, pat))

    return res
#@+node:ekr.20101004092958.6034: *6* get_functions
def get_functions(self, prefix = None):

    c = self.cursor()

    if prefix is not None:
        c.execute('select name, class, file, searchpattern from function where name like (?)',( prefix + '%',))
    else:
        c.execute('select name, class, file, searchpattern from function')

    return [(name, pat, klassid, fileid) for name, klassid, fileid, pat in c]
#@+node:ekr.20101004092958.6035: *6* file_id
def file_id(self, fname):
    if fname == '':
        return 0

    idd = self.filecache.get(fname)
    if idd is None:
        c = self.cursor()
        c.execute('insert into file(path) values (?)', [fname] )
        idd = c.lastrowid

        self.filecache[fname] = idd
        self.fileids_scanned.add(idd)
    else:
        if idd in self.fileids_scanned:
            return idd

        # we are rescanning a file with old entries - nuke old entries
        #print "rescan", fname
        c = self.cursor()
        c.execute("delete from function where file = (?)", (idd, ))
        #self.dbconn.commit()
        self.fileids_scanned.add(idd)

    return idd
#@+node:ekr.20101004092958.6036: *6* feed_function
def feed_function(self, func_name, class_name, file_name, aux):
    """ insert one function

    'aux' can be a search pattern (as with ctags), signature, or description.
    """
    clid = self.class_id(class_name)
    fid = self.file_id(file_name)
    c = self.cursor()
    c.execute('insert into function(class, name, searchpattern, file) values (?, ?, ?, ?)',
              [clid, func_name, aux, fid])
#@+node:ekr.20101004092958.6037: *6* feed_scintilla
def feed_scintilla(self, apifile_obj):
    """ handle scintilla api files

    Syntax is like:

    qt.QApplication.style?4() -> QStyle
    """

    for l in apifile_obj:
        if not isPython3:
            l = unicode(l, 'utf8', 'replace')
        parts = l.split('?')
        fullsym = parts[0].rsplit('.',1)
        klass, func = fullsym

        if len(parts) == 2:
            desc = parts[1]
        else:
            desc = ''

        # now our class is like qt.QApplication. We do the dirty trick and
        # remove all but actual class name

        shortclass = klass.rsplit('.',1)[-1]

        #print func, klass, desc
        self.feed_function(func.strip(), shortclass.strip(), '', desc.strip())
    self.dbconn.commit()
#@+node:ekr.20101004092958.6038: *6* feed_ctags
def feed_ctags(self,tagsfile_obj):
    for l in tagsfile_obj:
        #print l
        if not isPython3:
            l = unicode(l, 'utf8', 'replace')
        if l.startswith('!'):
            continue
        fields = l.split('\t')
        m = fields[0]
        fil = fields[1]
        pat = fields[2]
        typ = fields[3]
        klass = None
        try:
            ext = fields[4]
            if ext and ext.startswith('class:'):
                klass = ext.split(':',1)[1].strip()
                idd = self.class_id(klass)
                #print "klass",klass, idd

        except IndexError:
            ext = None
            # class id 0 = function
            idd = 0

        c = self.cursor()
        #print fields

        fid = self.file_id(fil)

        c.execute('insert into function(class, name, searchpattern, file) values (?, ?, ?, ?)',
                  [idd, m, pat, fid])

    self.dbconn.commit()
    #c.commit()
#@+node:ekr.20101004092958.6039: *6* add_source
def add_source(self, type, src):
    c = self.cursor()
    c.execute('insert into datasource(type, src) values (?,?)', (type,src))
    self.dbconn.commit()
#@+node:ekr.20101004092958.6040: *6* sources
def sources(self):
    c = self.cursor()
    c.execute('select type, src from datasource')
    return list(c)
#@+node:ekr.20101004092958.6041: *6* zap_symbols
def zap_symbols(self):
    c = self.cursor()
    tables = ['class', 'file', 'function']
    for t in tables:
        c.execute('delete from ' + t)
    self.dbconn.commit()
#@+node:ekr.20101004092958.6042: *6* parseall
# high level commands        
def parseall(self):
    sources = self.sources()
    self.reset_caches()
    self.zap_symbols()
    tagdirs = [td for typ, td in sources if typ == 'tagdir']
    self.parse(tagdirs)
    self.dbconn.commit()
#@+node:ekr.20101004092958.6043: *6* parse
def parse(self, paths):
    paths = set(os.path.abspath(p) for p in paths)
    f = run_ctags(paths)
    self.feed_ctags(f)
    sources = self.sources()
    for a in paths:
        if ('tagdir', a) not in sources:            
            self.add_source('tagdir', a)
#@+node:ekr.20101004092958.6044: *5* class ContextSniffer
class ContextSniffer:
    """ Class to analyze surrounding context and guess class

    For simple dynamic code completion engines

    """
    @others
#@+node:ekr.20101004092958.6045: *6* __init__
def __init__(self):
    # var name => list of classes
    self.vars = {}
#@+node:ekr.20101004092958.6046: *6* declare
def declare(self, var, klass):
    print("declare",var,klass)
    vars = self.vars.get(var, [])
    if not vars:
        self.vars[var] = vars
    vars.append(klass)
#@+node:ekr.20101004092958.6047: *6* push_declarations
def push_declarations(self, body):
    for l in body.splitlines():
        l = l.lstrip()
        if not l.startswith('#'):
            continue
        l = l.lstrip('#')
        parts = l.strip(':')
        if len(parts) != 2:
            continue
        self.declare(parts[0].strip(), parts[1].strip())
#@+node:ekr.20101004092958.6048: *6* set_small_context
def set_small_context(self, body):
    """ Set immediate function """
    self.push_declarations(body)
#@+node:ekr.20101004092958.6049: *4* codewise experiments
import codewise as cw
# print(dir(cw))

cw.cmd_init()
#@+node:ekr.20101004092958.6050: *4* How to make codewise work
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/ac3f8789010c882e/a1558a10eb8537c0?lnk=gst&q=codewise#a1558a10eb8537c0

1. Make sure you have exuberant ctags (not just regular ctags)
installed.  It's an Ubuntu package, so easy if you're using Ubuntu.

2. Install Ville's python module "codewise".  This is a small module on
which the Leo plugin relies.

   bzr branch lp:codewise
   cd codewise
   sudo python setup.py install

3. You need a recent trunk version of leo to get the plugin which uses
the above module.

4. Enable the plugin by putting "codewisecompleter.py" on an
uncommented line in your @enabled-plugins @settings node.

5. On the command line:

if you have an existing ~/.ctags for some reason, and it's nothing you
need to keep:

  rm ~/.ctags

then

  codewise setup
  codewise init
  codewise parse .../path/to/leo/  # assuming you want completion on
                                   # leo code
  codewise parse .../some/other/project/

Then, after restarting leo if necessary, type

c.op<Alt-0> in the body editor to find all the c. methods starting
with 'op' etc.

Nice work Ville, thanks.

==================

Thanks for this, I hope others will take a stab at it as well, given
sane instructions (I burned my free cycles frantically coding this
thing and neglected the all-important HOWTO). This is important
because functional completion is the single most important thing still
missing from Leo. Or, well, was ;-).

Especially the presentation part (QCompleter) needs some care, so you
can operate it from your keyboard alone. It should probably be moved
to core (qtgui, perhaps leoQTextEditWIdget), so codewise completer can
just invoke w.complete(list_of_completions) that will bring up the
QCompleter popup.

> Then, after restarting leo if necessary, type

> c.op<Alt-0> in the body editor to find all the c. methods starting
> with 'op' etc.

Also, try the explicit declarations:

# w : SomeClass

w.<alt+0>

And self.<alt+0> 
#@+node:ekr.20101025100847.5852: *3* @url ready to work on codewise
http://mail.google.com/mail/#inbox/12b7463d6c339406
#@+node:ekr.20101029092426.5861: *3* Finish QCompleter
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/ba5748aad35deba0

Here's my modest proposal for autocompletion work:

Please ensure that autocompletion works for c. and p. and g.

That's it.

The important thing is that it uses QCompleter and keyboard event
handling works. That's the hard part. Rest can be handled by plugins.
Every time the topic of autocompletion comes up, we end up solving the
"fun" problems like scanning and ctags, and the most important part
(gui) does not get done. 
#@+node:ekr.20101002192824.5843: *3* test-new-ac
# To do:
# - Fix findAnchor.
# - Report duck type in completion tab: callable if calltips exist.
# - What to do about library files and imports?

import re
import string
import sys
import time

@others

ac = NewAutoCompleter(c.k)
w = c.frame.body.bodyCtrl
event = g.bunch(char='e',keysym='e')
n = len(p.b)
if 0:
    w.setSelectionRange(n-7,n)
else:
    w.setInsertPoint(n)

ac.start(event=event,w=w)

# The cursor must be at the end
# p.s
#@+node:ekr.20101002192824.6019: *4* class AutoCompleterScanner
class AutoCompleterScanner:

    '''A class that scans for autocompleter completions.'''

    @others
#@+node:ekr.20101002192824.6020: *5*  ctor & helper (AutoCompleterScanner)
def __init__ (self,c):

    self.c = c

    # Set by definePatterns...
    self.okchars = None
    self.pats = {}

    # Set by scanners...
    self.calltips = {}
    self.watchwords = {}

    self.definePatterns()
#@+node:ekr.20101002192824.6021: *6* definePatterns
def definePatterns (self):

    self.space = r'[ \t\r\f\v ]+' # one or more whitespace characters.
    self.end = r'\w+\s*\([^)]*\)' # word (\w) ws ( any ) (can cross lines)

    # Define re patterns for various languages.
    # These patterns match method/function definitions.
    self.pats = {}
    self.pats ['python'] = re.compile(r'def\s+%s' % self.end)
        # def ws word ( any ) # Can cross line boundaries.
    self.pats ['java'] = re.compile(
        r'((public\s+|private\s+|protected\s+)?(static%s|\w+%s){1,2}%s)' % (
            self.space,self.space,self.end))
    self.pats ['perl'] = re.compile(r'sub\s+%s' % self.end)
    self.pats ['c++'] = re.compile(r'((virtual\s+)?\w+%s%s)' % (self.space,self.end))
    self.pats ['c'] = re.compile(r'\w+%s%s' % (self.space,self.end))

    # Define self.okchars.
    okchars = {}
    for z in string.ascii_letters:
        okchars [z] = z
    okchars ['_'] = '_'
    self.okchars = okchars 
#@+node:ekr.20101002192824.6022: *5* scan & dumps
def scan (self):

    '''Build the database from the outline'''

    trace = True and not g.unitTesting
    verbose = False
    if trace: t1 = time.time()
    c = self.c
    for p in c.all_unique_positions():
        self.scanForAutoCompleter(p.b)

    p = c.rootPosition()
    seen = {}
    while p:
        key = p.key()
        if not key in seen:
            seen[key] = True
            if p.isAnyAtFileNode():
                # scanForAtLanguage is expensive!
                language = g.scanForAtLanguage(c,p)
                for p2 in p.self_and_subtree():
                    seen[p2.key()] = True
                    self.scanForCallTip(p2.b,language)
        p.moveToThreadNext()

    if trace:
        t2 = time.time()
        if verbose:
            self.dumpTips()
            self.dumpWords()
        self.printSummary(t2-t1)

#@+node:ekr.20101002192824.6023: *6* dumpTips
def dumpTips (self):

    print('calltips...\n\n')
    for key in sorted(self.calltips):
        d = self.calltips.get(key)
        if d:
            if 1:
                d2 = {}
                for key2 in list(d.keys()):
                    aList = d.get(key2)
                    if len(aList) == 1:
                        s = aList[0]
                        if s.startswith(key2):
                            s = s[len(key2):].strip()
                        d2[key2] = s
                    else:
                        aList2 = []
                        for s in aList:
                            if s.startswith(key2):
                                s = s[len(key2):].strip()
                            aList2.append(s)
                        d2[key2] = aList2
                print('\n%s:\n\n' % (key),g.dictToString(d2))
            else:
                print('\n%s:\n\n' % (key),g.dictToString(d))
#@+node:ekr.20101002192824.6024: *6* dumpWords
def dumpWords (self):

    print('autocompleter words...\n\n')
    show = ['aList','c','d','f','g','gui','k','m','p','s','t','v','w']
    keys = show or sorted(self.watchwords)
    for key in keys:
        aList = self.watchwords.get(key)
        aList.sort()
        print('\n%s:' % (key),g.listToString(aList)) #repr(aList))
    print()
#@+node:ekr.20101002192824.6025: *6* printSummary
def printSummary (self,t):

    n  = len(list(self.watchwords.keys()))

    n2 = 0
    for language in list(self.calltips.keys()):
        d = self.calltips.get(language)
        n2 += len(list(d.keys()))

    g.es_print('scanned %s words, %s tips in %3.2fs sec.' % (
        n,n2,t))
#@+node:ekr.20101002192824.6026: *5* scanForAutoCompleter & helper
def scanForAutoCompleter (self,s):

    '''This function scans text for the autocompleter database.'''

    aList = []
    strings = s.split('.')
    if not strings: return

    for i in range(len(strings)-1):
        self.makeAutocompletionList(strings[i],strings[i+1],aList)

    if aList:
        for key,val in aList:
            aList2 = self.watchwords.get(key,[])
            # val = str(val)
            if val not in aList2:
                aList2.append(val)
                self.watchwords [key] = aList2
#@+node:ekr.20101002192824.6027: *6* makeAutocompletionList
def makeAutocompletionList (self,a,b,glist):

    '''We have seen a.b, where a and b are arbitrary strings.
    Append (a1.b1) to glist.
    To compute a1, scan backwards in a until finding whitespace.
    To compute b1, scan forwards in b until finding a char not in okchars.
    '''

    # Compute reverseFindWhitespace inline.
    i = len(a) -1
    while i >= 0:
        # if a[i].isspace() or a [i] == '.':
        if a[i] not in self.okchars:
            a1 = a [i+1:] ; break
        i -= 1
    else:
        a1 = a

    # Compute getCleanString inline.
    i = 0
    for ch in b:
        if ch not in self.okchars:
            b1 = b[:i] ; break
        i += 1
    else:
        b1 = b

    if b1:
        glist.append((a1,b1),)
#@+node:ekr.20101002192824.6028: *5* scanForCallTip & helper
def scanForCallTip (self,s,language):

    '''this function scans text for calltip info'''

    d = self.calltips.get(language,{})
    pat = self.pats.get(language or 'python')

    # Set results to a list of all the function/method defintions in s.
    results = pat and pat.findall(s) or []

    for z in results:
        if isinstance(z,tuple): z = z [0]
        if language == 'python':
            z = self.cleanPythonTip(z)
        pieces2 = z.split('(')
        # g.trace(pieces2)
        pieces2 [0] = pieces2 [0].split() [-1]
        a, junk = pieces2 [0], pieces2 [1]
        aList = d.get(a,[])
        if z not in aList:
            aList.append(z)
            d [a] = aList

    self.calltips [language] = d
#@+node:ekr.20101002192824.6029: *6* cleanPythonTip
def cleanPythonTip (self,s):

    result = []
    i = g.skip_ws(s,3)
    while i < len(s):
        ch = s[i]
        if ch in ' \t\n':
            i += 1
        elif ch == '#':
            # Remove comment.
            i += 1
            while i < len(s):
                i += 1
                if s[i-1] == '\n': break
        else:
            result.append(ch)
            i += 1
    return ''.join(result)
#@+node:ekr.20101002192824.5915: *4* class NewAutoCompleter (New)
class NewAutoCompleter:

    '''A class that inserts autocompleted and calltip text in text widgets.
    This class shows alternatives in the tabbed log pane.

    The keyHandler class contains hooks to support these characters:
    invoke-autocompleter-character (default binding is '.')
    invoke-calltips-character (default binding is '(')
    '''

    @others
#@+node:ekr.20101002192824.5916: *5*  ctor (autocompleter)
def __init__ (self,k):

    self.c = k.c
    self.k = k
    self.calltips = {}
        # Keys are language, values are dicts:
        # keys are ids, values are signatures.
    self.language = None
    self.leadinWord = None
    self.membersList = None
    self.prefix = None
    self.prefixes = []
    self.scanned = False
    self.selection = None # The selection range on entry.
    self.selectedText = None # The selected text on entry.
    self.tabList = []
    self.tabListIndex = -1
    self.tabName = None # The name of the main completion tab.
    self.useTabs = True # True: show results in autocompleter tab.
    self.verbose = False # True: print all members.
    self.watchwords = {} # Keys are ids, values are lists of ids that can follow a id dot.
    self.widget = None # The widget that should get focus after autocomplete is done.
#@+node:ekr.20101002192824.5919: *5* Top level
#@+node:ekr.20101002192824.5920: *6* autoComplete
def autoComplete (self,event=None,force=False):

    '''An event handler called from k.masterKeyHanderlerHelper.'''

    g.trace()

    c = self.c ; k = self.k ; gui = g.app.gui
    w = gui.eventWidget(event) or c.get_focus()

    # First, handle the invocation character as usual.
    k.masterCommand(event,func=None,stroke=None,commandName=None)

    # Allow autocompletion only in the body pane.
    if not c.widget_name(w).lower().startswith('body'):
        return 'break'

    self.language = g.scanForAtLanguage(c,c.p)
    if w and self.language == 'python' and (k.enable_autocompleter or force):
        self.start(event=event,w=w)

    return 'break'
#@+node:ekr.20101002192824.5921: *6* autoCompleteForce
def autoCompleteForce (self,event=None):

    '''Show autocompletion, even if autocompletion is not presently enabled.'''

    return self.autoComplete(event,force=True)
#@+node:ekr.20101002192824.5922: *6* autoCompleterStateHandler
def autoCompleterStateHandler (self,event):

    trace = False and not g.app.unitTesting
    c = self.c ; k = self.k ; gui = g.app.gui
    tag = 'auto-complete' ; state = k.getState(tag)
    ch = gui.eventChar(event)
    keysym = gui.eventKeysym(event)

    if trace: g.trace(repr(ch),repr(keysym),state)

    if state == 0:
        c.frame.log.clearTab(self.tabName)
        self.computeCompletionList()
        k.setState(tag,1,handler=self.autoCompleterStateHandler) 
    elif keysym in (' ','Return'):
        self.finish()
    elif keysym == 'Escape':
        self.abort()
    elif keysym == 'Tab':
        self.doTabCompletion()
    elif keysym in ('\b','BackSpace'): # Horrible hack for qt plugin.
        self.doBackSpace()
    elif keysym == '.':
        self.chain()
    ###elif keysym == '?':
    ###    self.info()
    elif keysym == '!':
        # Toggle between verbose and brief listing.
        self.verbose = not self.verbose
        self.computeCompletionList(verbose=self.verbose)
    elif ch and ch in string.printable:
        self.insertNormalChar(ch,keysym)
    else:
        # if trace: g.trace('ignore',repr(ch))
        return 'do-standard-keys'
#@+node:ekr.20101002192824.5924: *6* enable/disable/toggleAutocompleter/Calltips
def disableAutocompleter (self,event=None):
    '''Disable the autocompleter.'''
    self.k.enable_autocompleter = False
    self.showAutocompleterStatus()

def disableCalltips (self,event=None):
    '''Disable calltips.'''
    self.k.enable_calltips = False
    self.showCalltipsStatus()

def enableAutocompleter (self,event=None):
    '''Enable the autocompleter.'''
    self.k.enable_autocompleter = True
    self.showAutocompleterStatus()

def enableCalltips (self,event=None):
    '''Enable calltips.'''
    self.k.enable_calltips = True
    self.showCalltipsStatus()

def toggleAutocompleter (self,event=None):
    '''Toggle whether the autocompleter is enabled.'''
    self.k.enable_autocompleter = not self.k.enable_autocompleter
    self.showAutocompleterStatus()

def toggleCalltips (self,event=None):
    '''Toggle whether calltips are enabled.'''
    self.k.enable_calltips = not self.k.enable_calltips
    self.showCalltipsStatus()
#@+node:ekr.20101002192824.5925: *6* showCalltips
def showCalltips (self,event=None,force=False):

    '''Show the calltips at the cursor.'''

    c = self.c ; k = c.k ; w = g.app.gui.eventWidget(event)
    if not w: return

    # Insert the calltip if possible, but not in headlines.
    if (k.enable_calltips or force) and not c.widget_name(w).startswith('head'):
        self.widget = w
        self.prefix = ''
        self.selection = w.getSelectionRange()
        self.selectedText = w.getSelectedText()
        self.leadinWord = self.findCalltipWord(w)
        self.calltip()
    else:
        # Just insert the invocation character as usual.
        k.masterCommand(event,func=None,stroke=None,commandName=None)

    return 'break'
#@+node:ekr.20101002192824.5926: *6* showCalltipsForce
def showCalltipsForce (self,event=None):

    '''Show the calltips at the cursor, even if calltips are not presently enabled.'''

    return self.showCalltips(event,force=True)
#@+node:ekr.20101002192824.5927: *6* showAutocompleter/CalltipsStatus
def showAutocompleterStatus (self):
    '''Show the autocompleter status on the status line.'''

    k = self.k
    if not g.unitTesting:
        s = 'autocompleter %s' % g.choose(k.enable_autocompleter,'On','Off')
        g.es(s,color='red')

def showCalltipsStatus (self):
    '''Show the autocompleter status on the status line.'''
    k = self.k
    if not g.unitTesting:
        s = 'calltips %s' % g.choose(k.enable_calltips,'On','Off')
        g.es(s,color='red')
#@+node:ekr.20101002192824.5928: *5* Helpers
#@+node:ekr.20101002192824.5929: *6* .abort & exit (autocompleter)
def abort (self):

    k = self.k
    k.keyboardQuit(event=None,setDefaultStatus=False)
        # Stay in the present input state.
    self.exit(restore=True)

def exit (self,restore=False): # Called from keyboard-quit.

    k = self ; c = self.c 
    w = self.widget or c.frame.body.bodyCtrl
    for name in (self.tabName,'Modules','Info'):
        c.frame.log.deleteTab(name)
    c.widgetWantsFocusNow(w)
    i,j = w.getSelectionRange()
    if restore:
        if i != j: w.delete(i,j)
        w.insert(i,self.selectedText)
    w.setSelectionRange(j,j,insert=j)

    ### self.clear()
    ### self.theObject = None
#@+node:ekr.20101002192824.5930: *6* append/begin/popTabName
def appendTabName (self,word):

    self.setTabName(self.tabName + word + '.')

def beginTabName (self,word):

    # g.trace(word,g.callers())
    ### if word == 'self' and self.selfClassName:
    ###    word = '%s (%s)' % (word,self.selfClassName)
    self.setTabName('AutoComplete ' + word + '.')

def clearTabName (self):

    self.setTabName('AutoComplete ')

def popTabName (self):

    s = self.tabName
    i = s.rfind('.',0,-1)
    if i > -1:
        self.setTabName(s[0:i])

# Underscores are not valid in Pmw tab names!
def setTabName (self,s):

    c = self.c
    if self.useTabs:
        if self.tabName:
            c.frame.log.deleteTab(self.tabName)
        self.tabName = s.replace('_','') or ''
        c.frame.log.clearTab(self.tabName)
#@+node:ekr.20101002192824.5931: *6* appendToKnownObjects
def appendToKnownObjects (self,obj):

    if 0:
        if type(obj) in (types.InstanceType,types.ModuleType,types):
            if hasattr(obj,'__name__'):
                self.knownObjects[obj.__name__] = obj
                # g.trace('adding',obj.__name__)
#@+node:ekr.20101002192824.5997: *6* calltip
def calltip (self):

    c = self.c
    w = self.widget
    word = w.getSelectedText()
    language = g.scanForAtLanguage(c,c.p)
    d = self.calltips.get(language)
    s = '()'
    if d:
        aList = d.get(word,[])
        g.trace(word,repr(aList))
        if aList:
            s = aList[0]
    s = s.replace(word,'').replace('(self,','(').replace('(self)','()').strip()

    # insert the text and set j1 and j2
    junk,j = w.getSelectionRange() # Returns insert point if no selection.
    w.insert(j,s)
    c.frame.body.onBodyChanged('Typing')
    j1,j2 = j + 1, j + len(s)

    # End autocompletion mode, putting the insertion point after the suggested calltip.
    self.finish()
    c.widgetWantsFocusNow(w)
    w.setSelectionRange(j1,j2,insert=j2)
#@+node:ekr.20101002192824.5988: *6* chain
def chain (self):

    c = self.c ; w = self.widget
    word = w.getSelectedText()
    i,j = w.getSelectionRange()
    w.insert(j,'.')
    w.setInsertPoint(j+1)
    self.finish()
    self.start(chain=True)
#@+node:ekr.20101002192824.5940: *6* computeCompletionList
def computeCompletionList (self,verbose=False):

    c = self.c ; w = self.widget
    c.widgetWantsFocus(w)
    s = w.getSelectedText()
    self.tabList,common_prefix = g.itemsMatchingPrefixInList(
        s,self.membersList,matchEmptyPrefix=True)

    if not common_prefix:
        if verbose or len(self.tabList) < 25 or not self.useTabs:
            self.tabList,common_prefix = g.itemsMatchingPrefixInList(
                s,self.membersList,matchEmptyPrefix=True)
        else: # Show the possible starting letters.
            d = {}
            for z in self.tabList:
                ch = z and z[0] or ''
                if ch:
                    n = d.get(ch,0)
                    d[ch] = n + 1
            aList = [ch+'...%d' % (d.get(ch)) for ch in sorted(d)]
            self.tabList = aList

    if self.useTabs:
        c.frame.log.clearTab(self.tabName) # Creates the tab if necessary.
        if self.tabList:
            self.tabListIndex = -1 # The next item will be item 0.
            self.setSelection(common_prefix)
        g.es('','\n'.join(self.tabList),tabName=self.tabName)
#@+node:ekr.20101002192824.5941: *6* doBackSpace (autocompleter)
def doBackSpace (self):

    '''Cut back to previous prefix.'''

    s = self.prefixes and self.prefixes.pop() or ''
    self.setSelection(s)
    self.computeCompletionList()
#@+node:ekr.20101002192824.5942: *6* doTabCompletion (autocompleter)
def doTabCompletion (self):

    '''Handle tab completion when the user hits a tab.'''

    c = self.c ; w = self.widget
    s = w.getSelectedText()

    if s.startswith(self.prefix) and self.tabList:
        # g.trace('cycle','prefix',repr(self.prefix),len(self.tabList),repr(s))
        # Set the label to the next item on the tab list.
        self.tabListIndex +=1
        if self.tabListIndex >= len(self.tabList):
           self.tabListIndex = 0
        self.setSelection(self.tabList[self.tabListIndex])
    else:
        self.computeCompletionList()

    c.widgetWantsFocusNow(w)
#@+node:ekr.20101002192824.5943: *6* extendSelection
def extendSelection (self,s):

    '''Append s to the presently selected text.'''

    c = self.c ; w = self.widget
    c.widgetWantsFocusNow(w)

    i,j = w.getSelectionRange()
    w.insert(j,s)
    j += 1
    w.setSelectionRange(i,j,insert=j)
    c.frame.body.onBodyChanged('Typing')
#@+node:ekr.20101002192824.5944: *6* findCalltipWord
def findCalltipWord (self,w):

    i = w.getInsertPoint()
    s = w.getAllText()
    if i > 0:
        i,j = g.getWord(s,i-1)
        word = s[i:j]
        return word
    else:
        return ''
#@+node:ekr.20101002192824.5946: *6* finish
def finish (self):

    c = self.c ; k = c.k

    k.keyboardQuit(event=None,setDefaultStatus=False)
        # Stay in the present input state.

    for name in (self.tabName,'Modules','Info'):
        c.frame.log.deleteTab(name)

    c.frame.body.onBodyChanged('Typing')
    c.recolor()
    ### self.clear()
    ### self.theObject = None
#@+node:ekr.20101002192824.5948: *6* getLeadinWord & helper
def getLeadinWord (self,w):

    i,word = self.findAnchor(w)

    if word and not word.isdigit():
        self.setMembersList(word)
        self.beginTabName(word)
        self.leadinWord = word
        return True
    else:
        self.membersList = []
        self.leadinWord = None
        return False
#@+node:ekr.20101002192824.5945: *7* findAnchor
def findAnchor (self,w):

    '''
    Scan backward for the word before the last period.
    Returns (j,word) where j is a Python index.'''

    i = j = w.getInsertPoint()
    s = w.getAllText()

    # Scan backward for the next '.'
    while i > 1 and s[i-1] != '.':
        # g.trace(i-1,s[i-1])
        i -= 1

    if i <= 0 or s[i-1] != '.':
        return 0,''

    i,j = g.getWord(s,i-2)
    word = s[i:j]
    if word == '.':
        # g.trace('word is dot')
        return 0,''
    else:
        # g.trace(i,j,repr(word))
        return j,word
#@+node:ekr.20101002192824.5949: *6* getMembersList
def getMembersList (self,obj):

    '''Return a list of possible autocompletions for self.leadinWord.'''

    if obj:
        aList = inspect.getmembers(obj)
        members = ['%s:%s' % (a,g.prettyPrintType(b))
            for a,b in aList if not a.startswith('__')]
        members.sort()
        return members
    else:
        return []
#@+node:ekr.20101002192824.6005: *6* info (rewrite)
def info (self):

    c = self.c ; doc = None ; obj = self.theObject ; w = self.widget

    word = w.getSelectedText()

    if not word:
        # Never gets called, but __builtin__.f will work.
        word = self.findCalltipWord(w)
        if word:
            # Try to get the docstring for the Python global.
            f = __builtins__.get(self.leadinWord)
            doc = f and f.__doc__

    if not doc:
        if not self.hasAttr(obj,word):
            g.es('no docstring for',word,color='blue')
            return
        obj = self.getAttr(obj,word)
        doc = inspect.getdoc(obj)

    if doc:
        c.frame.log.clearTab('Info',wrap='word')
        g.es('',doc,tabName='Info')
    else:
        g.es('no docstring for',word,color='blue')
#@+node:ekr.20101002192824.5951: *6* insertNormalChar
def insertNormalChar (self,ch,keysym):

    k = self.k ; w = self.widget

    g.trace(ch,self.prefix)

    if g.isWordChar(ch):
        # Look ahead to see if the character completes any item.
        s = w.getSelectedText() + ch
        tabList,common_prefix = g.itemsMatchingPrefixInList(
            s,self.membersList,matchEmptyPrefix=True)
        #g.trace('tabList',repr(tabList))
        #g.trace('common_prefix',repr(common_prefix))
        if tabList:
            # Add the character.
            self.tabList = tabList
            self.extendSelection(ch)
            s = w.getSelectedText()
            if s.startswith(self.prefix):
                self.prefix = self.prefix + ch
            self.computeCompletionList()
    elif ch == '(':
        self.calltip()
    else:
        self.extendSelection(ch)
        self.finish()
#@+node:ekr.20101002192824.5952: *6* push, pop, clear, stackNames (To be deleted)
if 0:
    def push (self,obj):
        if obj is not None:
            self.prevObjects.append(obj)

    def pop (self):
        obj = self.prevObjects.pop()
        return obj

    def clear (self):
        self.prevObjects = []

    def stackNames (self):
        aList = []
        for z in self.prevObjects:
            if hasattr(z,'__name__'):
                aList.append(z.__name__)
            elif hasattr(z,'__class__'):
                aList.append(z.__class__.__name__)
            else:
                aList.append(str(z))
        return aList
#@+node:ekr.20101002192824.5953: *6* setMembersList & helpers
def setMembersList (self,word):

    self.membersList = self.watchwords.get(word)
    self.membersList.sort()

    # g.trace(word,self.membersList)
#@+node:ekr.20101002192824.5954: *7* getObjectFromAttribute
def getObjectFromAttribute (self,word):

    obj = self.theObject

    if obj and self.hasAttr(obj,word):
        self.push(self.theObject)
        self.theObject = self.getAttr(obj,word)
        self.appendToKnownObjects(self.theObject)
        self.membersList = self.getMembersList(self.theObject)
    else:
        # No special support for 'self' here.
        # Don't clear the stack here!
        self.membersList = []
        self.theObject = None
#@+node:ekr.20101002192824.5956: *7* completeFromObject
def completeFromObject (self,obj):

    if obj:
        self.appendToKnownObjects(obj)
        self.push(self.theObject)
        self.theObject = obj
        self.membersList = self.getMembersList(obj=obj)
    else:
        self.theObject = None
        self.clear()
        self.membersList = []
#@+node:ekr.20101002192824.5957: *6* setSelection
def setSelection (self,s):

    g.trace(s,g.callers(2))

    c = self.c ; w = self.widget
    c.widgetWantsFocusNow(w)

    if w.hasSelection():
        i,j = w.getSelectionRange()
        w.delete(i,j)
    else:
        i = w.getInsertPoint()

    # Don't go past the ':' that separates the completion from the type.
    n = s.find(':')
    if n > -1: s = s[:n]

    w.insert(i,s)
    j = i + len(s)
    w.setSelectionRange(i,j,insert=j)

    # New in Leo 4.4.2: recolor immediately to preserve the new selection in the new colorizer.
    c.frame.body.recolor(c.p,incremental=True)
    # Usually this call will have no effect because the body text has not changed.
    c.frame.body.onBodyChanged('Typing')
#@+node:ekr.20101002192824.5958: *6* start
def start (self,event=None,w=None,prefix=None,chain=False):

    c = self.c
    if w: self.widget = w
    else: w = self.widget

    # We wait until now to define these dicts so that more classes and objects will exist.
    if not self.scanned:
        self.scanned = True
        # scanner = leoKeys.AutoCompleterScanner(c)
        scanner = AutoCompleterScanner(c)
        scanner.scan()
        self.calltips = scanner.calltips
        self.watchwords = scanner.watchwords

    self.selection = w.getSelectionRange()
    self.prefix = prefix or w.getSelectedText()
    if chain:
        if self.prefix not in self.prefixes:
            self.prefixes.append(self.prefix)
    else:
        self.prefixes = [self.prefix]
    self.selectedText = w.getSelectedText()
    flag = self.getLeadinWord(w)
    # g.trace(flag,self.membersList)
    if self.membersList:
        if not flag:
            # Remove the (leading) invocation character.
            i = w.getInsertPoint()
            s = w.getAllText()
            if i > 0 and s[i-1] == '.':
                s = g.app.gui.stringDelete(s,i-1)
                w.setAllText(s)
                c.frame.body.onBodyChanged('Typing')
        if self.useTabs:
            self.autoCompleterStateHandler(event)
        else:
            self.computeCompletionList()
            return self.tabList
    else:
        self.abort()
#@+node:ekr.20080603052650.466: ** Vim stuff
@nocolor

4.8 vim, cleanups, your mission...

http://groups.google.com/group/leo-editor/browse_thread/thread/141690c553bfde55

Vim mode users: your top 3 complaints, please

@color
#@+node:ekr.20100827114047.5895: *3* Bugs
#@+node:ekr.20101024062147.5994: *4* Minor
#@+node:ekr.20101021101942.6011: *5* Window size not honored when opened from menu
#@+node:ekr.20100522090453.5912: *5* Fix bugs with @verbatim, @raw and @end_raw
@nocolor-node

> In the @file family of derived files (@thin, @nosent, etc.)  (i.e.,
> not @root), you can prevent any interpretation of the immediately
> following line with the "@verbatim" directive

This is not true, although it might appear to be true.

There is an @verbatim *sentinel*, but no @verbatim *directive*.

For example, I have just verified that the following does not work:

@verbatim
<< undefined ref >>

@raw and @end_raw do work as described.

There appear to be several bugs in this area:

1. Leo probably should warn that @verbatim does not exist as a
directive, although technically Leo is supposed to write anything that
looks like a sentinel but isn't "verbatim".

2. Leo does appear to read @raw and @end_raw properly in @thin files,
but Leo improperly issues the (red) log message,
converting @file format in @thin test_verbatim_sentinel.py
#@+node:ekr.20100223133351.5998: *5* show-invisibles doesn't work for blank lines
#@+node:ekr.20081208102356.1: *5* Threading colorizer doesn't handle multiple body editors
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/5be7a099b299327e

> Tk only colorizes one body editor, and if you delete that editor it
> colorizes no editor.

Thanks for this report.  This is a problem, never noticed until now,
with the threading colorizer.  A workaround is to disable the
threading colorizer plugin. 
#@+node:ekr.20100521090440.5885: *5* Fix mac unicode crasher
#@+node:ekr.20100521090440.5886: *6* Report
@nocolor-node

I put the following in qtGui.py after

# Last minute-munges to keysym.

    if sys.platform.startswith('darwin'):
        darwinmap = {
            'Alt-Key-5': '[',
            'Alt-Key-6': ']',
            'Alt-Key-7': '|',
            'Alt-slash': '\\',
            'Alt-Key-8': '{',
            'Alt-Key-9': '}',
            'Alt-e': '€',
            'Alt-l': '@',
        }
        if tkKey in darwinmap:
            tkKey = stroke = darwinmap[tkKey]

It works quite reasonabe. Sometimes (not really well reproducable)
following message appears:

/Users/tst/Leo-4.7-final/leo/core/leoKeys.py:3363: UnicodeWarning:
Unicode equal comparison failed to convert both arguments to Unicode -
interpreting them as being unequal
  if k.abortAllModesKey and stroke == k.abortAllModesKey:
/Users/scalet/Leo-4.7-final/leo/core/leoKeys.py:2521: UnicodeWarning:
Unicode equal comparison failed to convert both arguments to Unicode -
interpreting them as being unequal
  if k.abortAllModesKey and stroke == k.abortAllModesKey: # 'Control-
g'

nonetheless the special characters were inserted as expected.

I think all this is quite a hack, but ... thanks for the hack, most
important for me it's working for now. 
#@+node:ekr.20101024062147.6024: *5* bug 622819: Ctrl-Shift movement is incorrect
@nocolor-node

Ctrl-Shift movement (back-word-extend-selection, forward-word-extend-selection)
is incorrect when there is an existing selection, which was created by some
method other than Ctrl-Shift movement.

Expected behavior: it should extend/contract the existing selection

Actual behavior: it creates a new selection from the cursor position

======

This is a minor bug, but hard to fix because Leo doesn't always distinguish
between the point and the mark.

The fix would be in extendHelper, called from moveToHelper.
#@+node:ekr.20101024062147.5995: *4* Rst problems
#@+node:ekr.20101021160326.5948: *5* Fix rst3 unbounded recursion
@nocolor-node

on doing an Alt-X rst3 command

Traceback (most recent call last):
  File "leoCommands.py", line 359, in doCommand
    val = command(event)
  File "leoCommands.py", line 764, in minibufferCallback
    retval = function(keywords)
  File "mod_scripting.py", line 642, in __call__
    self.controller.executeScriptFromButton(self.p,self.b,self.buttonText)
  File "mod_scripting.py", line 667, in executeScriptFromButton
    c.executeScript(args=args,p=p,silent=True)
  File "leoCommands.py", line 1991, in executeScript
    script = g.getScript(c,p,useSelectedText=useSelectedText)
  File "leoGlobals.py", line 4358, in getScript
    at = leoAtFile.atFile(c)
  File "leoAtFile.py", line 145, in __init__
    'check-python-code-on-write',default=True)
  File "leoCommands.py", line 7436, in getBool
    return g.app.config.getBool(self.c,setting,default=default)
  File "leoConfig.py", line 1445, in getBool
    val = self.get(c,setting,"bool")
  File "leoConfig.py", line 1337, in get
    isLeoSettings = c and c.shortFileName().endswith('leoSettings.leo')
  File "leoCommands.py", line 6673, in shortFileName
    return g.shortFileName(self.mFileName)
  File "leoGlobals.py", line 2548, in shortFileName
    return g.os_path_basename(fileName)
  File "leoGlobals.py", line 3401, in os_path_basename
    path = g.toUnicodeFileEncoding(path)
  File "leoGlobals.py", line 3643, in toUnicodeFileEncoding
    return g.toUnicode(path)
  File "leoGlobals.py", line 4636, in toUnicode
    elif mustConvert(s):
  File "leoGlobals.py", line 4632, in mustConvert
    return type(s) != types.UnicodeType
RuntimeError: maximum recursion depth exceeded while calling a Python object
#@+node:ekr.20100827114047.5893: *5* Fix rst file problems
@nocolor-node

Reported by TL:

When editing a Docutils file in an external editor (exported from
Leo), I always get a "Replace changed outline with external changes?"
dialog box when I save the file in the external editor. This appears
to occur because a newline is added to the end of the file when it is
imported back into Leo.

Note: The addition of a newline occurs even if the imported file
already has a newline at the end of the file.

There appears to be two problems:

1) Leo always adds a newline even if the already has one.
2) After Leo has added the newline it mistakenly marks the buffer as
modified.
#@+node:ekr.20100220190251.5616: *6* Fix rst bug?
# See handleCodeMode
#@+node:ekr.20100220190251.5617: *7* report
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/c212814815c92aac

The indentation of the second @doc causes the doc part to look like
a continuation of the code:: directive.

It's not clear whether this bug is worth fixing, and it's not clear whether
there would be side effects of any potential fix.

The simplest fix appears to be in handleCodeMode.
#@+node:ekr.20090502071837.72: *7* handleCodeMode & helper
def handleCodeMode (self,lines):

    '''Handle the preprocessed body text in code mode as follows:

    - Blank lines are copied after being cleaned.
    - @ @rst-markup lines get copied as is.
    - Everything else gets put into a code-block directive.'''

    result = [] ; n = 0 ; code = []
    while n < len(lines):
        s = lines [n] ; n += 1
        if (
            self.isSpecialDocPart(s,'@rst-markup') or (
                self.getOption('show_doc_parts_as_paragraphs') and
                self.isSpecialDocPart(s,None)
            )
        ):
            if code:
                self.finishCodePart(result,code)
                code = []
            result.append('')
            n, lines2 = self.getDocPart(lines,n)
            # A fix, perhaps dubious, to a bug discussed at
            # http://groups.google.com/group/leo-editor/browse_thread/thread/c212814815c92aac
            # lines2 = [z.lstrip() for z in lines2]
            # g.trace('lines2',lines2)
            result.extend(lines2)
        elif not s.strip() and not code:
            pass # Ignore blank lines before the first code block.
        elif not code: # Start the code block.
            result.append('')
            result.append(self.code_block_string)
            code.append(s)
        else: # Continue the code block.
            code.append(s)

    if code:
        self.finishCodePart(result,code)
        code = []

    # Munge the result so as to keep docutils happy.
    # Don't use self.rstripList: it's not the same.
    # g.trace(result)
    result2 = []
    for z in result:
        if z == '': result2.append('\n\n')
        # 2010/08/27: Fix bug 618482.
        # elif not z.rstrip(): pass
        elif z.endswith('\n\n'): result2.append(z) # Leave alone.
        else: result2.append('%s\n' % z.rstrip())

    return result2
#@+node:ekr.20090502071837.73: *8* formatCodeModeLine
def formatCodeModeLine (self,s,n,numberOption):

    if not s.strip(): s = ''

    if numberOption:
        return '\t%d: %s' % (n,s)
    else:
        return '\t%s' % s
#@+node:ekr.20090502071837.74: *8* rstripList
def rstripList (self,theList):

    '''Removed trailing blank lines from theList.'''

    # 2010/08/27: fix bug 618482.
    s = ''.join(theList).rstrip()
    return s.split('\n')
#@+node:ekr.20090502071837.75: *8* finishCodePart
def finishCodePart (self,result,code):

    numberOption = self.getOption('number_code_lines')
    code = self.rstripList(code)
    i = 0
    for line in code:
        i += 1
        result.append(self.formatCodeModeLine(line,i,numberOption))
#@+node:ekr.20100220204150.5627: *7* @@rst hello.html
@ @rst-options
verbose=True
code_mode=True
show_doc_parts_as_paragraphs=True
number_code_lines=True
write_intermediate_file=True
@c

#@+node:ekr.20100220204150.5628: *8* Greet the world
@
Greet the world, politely
@c

g.es ("Hello, world")

@
    Was that polite enough?
    Indentation.
@c
#@+node:ekr.20100826110728.5840: *5* rst with show-comments-as-paragraph
@nocolor-node

derwish

It seems that this option only works as expected for comments at the
beginning of a node.

A Leo file with the following contents would be rendered with  the
second comment inside the PRE-element of the code paragraph.

I am using Leo 4.7b2, build 2404

<?xml version="1.0" encoding="utf-8"?>
<?xml-stylesheet ekr_test?>
<leo_file>
<leo_header file_format="2" tnodes="0" max_tnode_index="0"
clone_windows="0"/>
<globals body_outline_ratio="0.5">
       <global_window_position top="30" left="12" height="600" width="800"/>
       <global_log_window_position top="0" left="0" height="0" width="0"/>
</globals>
<preferences/>
<find_panel_settings/>
<vnodes>
<v t="huesingjohannes.20090706083342.1221"
str_leo_pos="2,0"><vh>@chapters</vh></v>
<v t="huesingjohannes.20091022082006.1227"><vh>@url U:/leo/hello.html</
vh></v>
<v t="huesingjohannes.20090706083342.1220" a="E"><vh>@rst U:\leo
\hello.html</vh>
<v t="huesingjohannes.20090706083342.5924"><vh>Greet the world</vh></
v>
</v>
</vnodes>
<tnodes>
<t tx="huesingjohannes.20090706083342.1220">@ @rst-options
verbose=True
code_mode=True
show_doc_parts_as_paragraphs=True
number_code_lines=False

@c

</t>
<t tx="huesingjohannes.20090706083342.1221"></t>
<t tx="huesingjohannes.20090706083342.5924">@
    Greet the world, politely
@c

g.es ("Hello, world")

@
   Was that polite enough?
@c</t>
<t tx="huesingjohannes.20091022082006.1227"></t>
</tnodes>
</leo_file>

========================

Is there anyone here who could help me on this? It would be really nice if I
didn't have to restrain myself to comments only at the top of a node.


==========================
Thanks for this report.  Sorry for the delay in responding.  Please
file an official bug report.  It does seem like a real bug.


If I can fix it cleanly tomorrow before Leo 4.7 final I shall. Otherwise soon.

=======================
Ok. I have simple fixes, but none are safe enough to include in Leo 4.7 final...

In the meantime, removing the leading whitespace from the first line
of the second @doc part should be a good enough workaround.

There are three possible fixes, all similar:

1. In getDocPart (in leoRst.py), change::

   result.append(s)

to:

   result.append(s.lstrip())

There are two such lines in getDocPart.  Probably both should be
changed.

2. The same fix, conditional on a new 'stripLines' keyword arg.  Like
this:

   if stripLines:
       result.append(s.lstrip())
   else:
       result.append(s)

Again, this change affects two lines in getDocPart.

The idea is that handleCodeMode would be the only method to set the
stripLines arg:

   n, lines2 = self.getDocPart(lines,n)

3. A change to handleCodeMode.  After the line:

   n, lines2 = self.getDocPart(lines,n)

add the line:

   lines2 = [z.lstrip() for z in lines2]

As I write this, I am thinking that change #3 is the simplest change,
with the fewest side effects.

We can test any one of these solutions starting early in the Leo 4.8
release cycle, but none  is safe enough to include in Leo 4.7 final.

If you would, please test fix #3 and report your results.  Thanks.
#@+node:ekr.20101024062147.5996: *4* Wishlist
#@+node:ekr.20080922115725.1: *5* Finish @shadow
# Allow block comments in private shadow files.
# Compute delims using the private shadow file, not the file extension!
# Can @shadow mark externally changed nodes?
#@+node:ekr.20081004102201.2: *6* Log file for @shadow
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/5e7bd3af2d1fbf51

How about a shadow.log file which Leo told what it thought of the relationship
between the node, file and shadow? It might provide useful clues.
#@+node:ekr.20081001062423.1: *6* Can @shadow mark externally changed nodes?
@nocolor

http://groups.google.com/group/leo-editor/browse_thread/thread/c46eabe8a9fe6e8

@color
#@+node:ekr.20080708094444.38: *7* x.propagate_changed_lines
def propagate_changed_lines(self,new_public_lines,old_private_lines,marker,p=None):

    '''Propagate changes from 'new_public_lines' to 'old_private_lines.

    We compare the old and new public lines, create diffs and
    propagate the diffs to the new private lines, copying sentinels as well.

    We have two invariants:
    1. We *never* delete any sentinels.
       New at 2010/01/07: Replacements preserve sentinel locations.
    2. Insertions that happen at the boundary between nodes will be put at
       the end of a node.  However, insertions must always be done within sentinels.
    '''

    trace = False and g.unitTesting
    verbose = True
    x = self
    # mapping tells which line of old_private_lines each line of old_public_lines comes from.
    old_public_lines, mapping = self.strip_sentinels_with_map(old_private_lines,marker)

    << init vars >>
    << define print_tags >>

    delim1,delim2 = marker.getDelims()
    sm = difflib.SequenceMatcher(None,old_public_lines,new_public_lines)
    prev_old_j = 0 ; prev_new_j = 0

    for tag,old_i,old_j,new_i,new_j in sm.get_opcodes():

        << About this loop >>

        # Verify that SequenceMatcher never leaves gaps.
        if old_i != prev_old_j: # assert old_i == prev_old_j
            x.error('can not happen: gap in old: %s %s' % (old_i,prev_old_j))
        if new_i != prev_new_j: # assert new_i == prev_new_j
            x.error('can not happen: gap in new: %s %s' % (new_i,prev_new_j))

        << Handle the opcode >>

        # Remember the ends of the previous tag ranges.
        prev_old_j = old_j
        prev_new_j = new_j

    # Copy all unwritten sentinels.
    self.copy_sentinels(
        old_private_lines_rdr,
        new_private_lines_wtr,
        marker,
        limit = old_private_lines_rdr.size())

    # Get the result.
    result = new_private_lines_wtr.getlines()
    if 1:
        << do final correctness check>>
    return result
#@+node:ekr.20080708094444.40: *8* << init vars >>
new_private_lines_wtr = self.sourcewriter(self)
# collects the contents of the new file.

new_public_lines_rdr = self.sourcereader(self,new_public_lines)
    # Contains the changed source code.

old_public_lines_rdr = self.sourcereader(self,old_public_lines)
    # this is compared to new_public_lines_rdr to find out the changes.

old_private_lines_rdr = self.sourcereader(self,old_private_lines) # lines_with_sentinels)
    # This is the file which is currently produced by Leo, with sentinels.

# Check that all ranges returned by get_opcodes() are contiguous
old_old_j, old_i2_modified_lines = -1,-1

tag = old_i = old_j = new_i = new_j = None
#@+node:ekr.20080708094444.39: *8* << define print_tags >>
def print_tags(tag, old_i, old_j, new_i, new_j, message):

    sep1 = '=' * 10 ; sep2 = '-' * 20

    g.pr('\n',sep1,message,sep1,p and p.h)

    g.pr('\n%s: old[%s:%s] new[%s:%s]' % (tag,old_i,old_j,new_i,new_j))

    g.pr('\n',sep2)

    table = (
        (old_private_lines_rdr,'old private lines'),
        (old_public_lines_rdr,'old public lines'),
        (new_public_lines_rdr,'new public lines'),
        (new_private_lines_wtr,'new private lines'),
    )

    for f,tag in table:
        f.dump(tag)
        g.pr(sep2)


#@+node:ekr.20080708192807.2: *8* << about this loop >>
@ This loop writes all output lines using a single writer:
new_private_lines_wtr.

The output lines come from two, and *only* two readers:

1. old_private_lines_rdr delivers the complete original sources. All
   sentinels and unchanged regular lines come from this reader.

2. new_public_lines_rdr delivers the new, changed sources. All inserted or
   replacement text comes from this reader.

Each time through the loop, the following are true:

- old_i is the index into old_public_lines of the start of the present
  SequenceMatcher opcode.

- mapping[old_i] is the index into old_private_lines of the start of
  the same opcode.

At the start of the loop, the call to copy_sentinels effectively skips
(deletes) all previously unwritten non-sentinel lines in
old_private_lines_rdr whose index is less than mapping[old_i].

As a result, the opcode handlers do not need to delete elements from
the old_private_lines_rdr explicitly. This explains why opcode
handlers for the 'insert' and 'delete' opcodes are identical.
#@+node:ekr.20080708192807.5: *8* << Handle the opcode >>
# Do not copy sentinels if a) we are inserting and b) limit is at the end of the old_private_lines.
# In this special case, we must do the insert before the sentinels.
limit=mapping[old_i]

if trace: g.trace('tag',tag,'old_i',old_i,'limit',limit)

if tag == 'equal':
    # Copy sentinels up to the limit = mapping[old_i]
    self.copy_sentinels(old_private_lines_rdr,new_private_lines_wtr,marker,limit=limit)

    # Copy all lines (including sentinels) from the old private file to the new private file.
    start = old_private_lines_rdr.index() # Only used for tag.
    while old_private_lines_rdr.index() <= mapping[old_j-1]:
        line = old_private_lines_rdr.get()
        new_private_lines_wtr.put(line,tag='%s %s:%s' % (
            tag,start,mapping[old_j-1]))

    # Ignore all new lines up to new_j: the same lines (with sentinels) have just been written.
    new_public_lines_rdr.sync(new_j)

elif tag == 'insert':
    if limit < old_private_lines_rdr.size():
        self.copy_sentinels(old_private_lines_rdr,new_private_lines_wtr,marker,limit=limit)
    # All unwritten lines from old_private_lines_rdr up to mapping[old_i] have already been ignored.
    # Copy lines from new_public_lines_rdr up to new_j.
    start = new_public_lines_rdr.index() # Only used for tag.
    while new_public_lines_rdr.index() < new_j:
        line = new_public_lines_rdr.get()
        if marker.isSentinel(line):
            new_private_lines_wtr.put(
                '%s@verbatim%s\n' % (delim1,delim2),
                tag='%s %s:%s' % ('new sent',start,new_j))
        new_private_lines_wtr.put(line,tag='%s %s:%s' % (tag,start,new_j))

elif tag == 'replace':
    # This case is new: it was the same as the 'insert' case.
    start = old_private_lines_rdr.index() # Only used for tag.
    while (
        old_private_lines_rdr.index() <= mapping[old_j-1]
        and new_public_lines_rdr.index() <  new_j
            # 2010/10/22: the replacement lines can be shorter.
    ):
        old_line = old_private_lines_rdr.get()
        if marker.isSentinel(old_line):
            # Important: this should work for @verbatim sentinels
            # because the next line will also be replaced.
            new_private_lines_wtr.put(old_line,tag='%s %s:%s' % (
                'replace: copy sentinel',start,new_j))
        else:
            new_line = new_public_lines_rdr.get()
            new_private_lines_wtr.put(new_line,tag='%s %s:%s' % (
                'replace: new line',start,new_j))

    # 2010/10/22: The replacement lines can be longer: same as 'insert' code above.
    while new_public_lines_rdr.index() < new_j:
        line = new_public_lines_rdr.get()
        if marker.isSentinel(line):
            new_private_lines_wtr.put(
                '%s@verbatim%s\n' % (delim1,delim2),
                tag='%s %s:%s' % ('new sent',start,new_j))
        new_private_lines_wtr.put(line,tag='%s %s:%s' % (tag,start,new_j))

elif tag=='delete':
    # Copy sentinels up to the limit = mapping[old_i]
    self.copy_sentinels(old_private_lines_rdr,new_private_lines_wtr,marker,limit=limit)
    # Leave new_public_lines_rdr unchanged.

else: g.trace('can not happen: unknown difflib.SequenceMather tag: %s' % repr(tag))

if trace and verbose:
    print_tags(tag, old_i, old_j, new_i, new_j, "After tag")
#@+node:ekr.20080708094444.45: *8* << do final correctness check >>
t_sourcelines, t_sentinel_lines = self.separate_sentinels(
    new_private_lines_wtr.lines, marker)

self.check_the_final_output(
    new_private_lines   = result,
    new_public_lines    = new_public_lines,
    sentinel_lines      = t_sentinel_lines,
    marker              = marker)
#@+node:ekr.20090402072059.13: *6* Create a general mechanism for aux (shadow, _db) files
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/4ec30df3f1db8db3

On Sat, Mar 28, 2009 at 3:24 AM, VR <viktor.ransmayr@gmail.com> wrote:


    When I tried to de-install Leo-4.6b1 I succeeded, but the program
    reported that 5 directories
    were not removed.

    Three of the directories where

    1) C:\Python26\Lib\site-packages\Leo-4-6-b1\leo\config
    2) C:\Python26\Lib\site-packages\Leo-4-6-b1\leo\doc
    3) C:\Python26\Lib\site-packages\Leo-4-6-b1\leo\plugins

    [containing]


    a) .leoSettings.leo_db
    b) .leoDocs.leo_db
    c) .leo_shadow
    d) .leoPluginsRef.leo_db


Thanks for this report. I think it is important, and needs a good solution.

I dislike all these files being sprayed around the file system. I'd like to see
these files placed somewhere the ~/.leo directory. Is there a reason why this
would be a bad idea?

Similarly, we might also prefer to have shadow files place in, say,
~/.leo/shadow_files.

In both cases, I think we want to create files that indicate their location.
Either that, or mirror their location in (subdirectories) ~/.leo. In other
words, this is a general problem, and it would be good to have a robust, general
solution.
#@+node:ekr.20100131161507.6303: *5* Unit tests that all commands have docstrings
# Just make the test.  It doesn't have to pass.
#@+node:ekr.20100826110728.5839: *5* Relocating .leo_shadow directories
@nocolor-node

2008

http://groups.google.com/group/leo-editor/browse_thread/thread/b738e3f8d164f9fc

May 10, 2010

Kent

I think there could be quite a bit of interest in moving
the shadow files to their own tree, avoiding what might
be considered 'pollution' of a tree of files in @shadow nodes.

Edward has said that this would add a lot of complexity to Leo.

It seems that a VCS back end for Leo might simplify the
task of arbitrary shadow file location, as well as adding
versioning capability.


Those of us old enough to remember the Groucho Marx show will know what I am ...

bogomil
 to me

In order to relocate .leo_shadow directories in home dir, I have made
the following changes leoShadow.py:
1. Introduce new setting 'shadow_in_home_dir':
   x.ctor:
     ...
     self.shadow_prefix = c.config.getString('shadow_prefix') or ''

=>  self.shadow_in_home_dir = c.config.getBool('shadow_in_home_dir')
or False
    ...

2. Make the following line:
   x.shadowDirName and shadowPathName:
      ...
      fileDir = g.os_path_dirname(filename)

=>   if self.shadow_in_home_dir:
        fileDir = "//".join([baseDir, fileDir.replace(':','%')])

In this way I keep .leo_shadow dirs in a tree and it is ok if the user
reorgs the original tree.

=====================
Edward K. Ream
 to bogomil

Thanks for these suggestions.  I'll try them soon.

A few minor comments about the code.

>  self.shadow_in_home_dir = c.config.getBool('shadow_in_home_dir') or False

This works (because c.config.getBool returns None if the setting does
not exist). Thus, the "or False" part merely replaces None by False.
I prefer the following:

self.shadow_in_home_dir = c.config.getBool('shadow_in_home_dir',default=False)

>   if self.shadow_in_home_dir:
>         fileDir = "//".join([baseDir, fileDir.replace(':','%')])

This looks like a Windows-only solution because of ':'.  It might fail
in strange ways on other platforms.
#@+node:ekr.20100830120622.5829: *5* Fix python import problems
@nocolor-node

> > Hmm, I guess that would be more clear, although I think I'd like an option
to include it in the following def to avoid

> > Decoration
> > index
> > Decoration
> > add_user
>
> Sure.  Decorations must always be part of a definition.

Well, personally I'd like to have them included in the definition, but I think
Kent's preference for a separate node is reasonable to. If your function and
hence definition node is called "pluralize", and it's decorated with something
like "@authenticate_user", you may never check the innocent looking pluralize
definition to find out what on earth's triggering the mysterious network
database call. And this isn't a completely specious example, authentication may
have been added to stop pluralize being used in a user existence detection
exploit or something. OTOH in well behaved code like CherryPy apps you don't
want a separate node for every @cherrypy.expose.

Bottom line is I think we're asking for a set of @settings to fine tune python
import behavior:

@python_import_interdef = previous | next | own_node | ai
@python_import_decoration = next | own_node

I'm not sure I believe the AI option is possible / practical, and am not asking
for it, just listing it :-)

I'd also like

@python_import_init_in_class_node = true | false

as often there's more docs on a class in the __init__ than the class docstring.

I think that's really all we're talking about, some @settings to test during import.
#@+node:ekr.20100828074347.5827: *3* Vim bindings
#@+node:ekr.20100521090440.5887: *4* Generalize minibuffer code
Just playing with part of a template system, the (partial) mock up for
"range()" is:

def tabStopNaming (event=None):

  stateName = 'naming'
  k = c.k
  state = k.getState(stateName)

  help = ('start-value -- optional, -> fill-in or tab eliminate.  ',
           'end-value -- required, -> fill-in.  ',
           'step -- optional, ->fill-in or tab to eliminate.  ')
  tabStop = ('start-value', 'end-value', 'step')

  if state == 0:
      k.setLabelBlue(help[0],protect=True)
      k.getArg(event,stateName,1,tabStopNaming)
      # g.es('does this ever executed?') # yes, imediately!
  else:
      k.clearState()
      g.es_print('%s : %s' % (tabStop[0], k.arg))
      k.setLabelBlue('')

tabStopNaming()

=====================================================
This is hardwired for the first parameter.  Things I need to expand
this:

1. put in a variable that cycles through the tabStops

2. In this mock up, you are entering the parameters in the minibuffer,
a more advanced version would collect each keypress and put it in the
body at the current tabStop, a tab would finalize the entry and
advance to the next stop, no text other than the 'help', ends up in
the minibuffer.

Sinc I'm only modifying existing code without real understanding of
what Leo is doing, andy guidance would be appreciated.

Tom

=======================

Thanks for this work.  You obviously understand the present code well
enough to play with it.

Generalizing the minibuffer code would be a good thing to do.  I
suppose we could define some kind of language that would create, or
rather simulate, hand-written code.  In a sense, @mode nodes are
intended to do this also.

As I look at leoKeys.py again I'll keep this example in the back of my
mind as something that it would be good to support more easily.

Edward
#@+node:ekr.20100210102224.5744: *4* Notes for key bindings
@nocolor-node

Requirements:
    - Existing bindings must still be valid.
    - But @mode does not have to remain.

Think data:
    - Drive binding by tables.
    *** Design these tables first.
    - Table design should be flixible.
      It can change without affecting user.

keySequence class?
    - Represents a sequence of keystrokes.

bufferMode class?
    - Might encapsulate directives, etc.

bindHelper class?
    - Would mediate various aspects of binding.
    - c.bindHelper or k.bindHelper?
#@+node:ekr.20060927173836.6: *4* Don't abort mode if there are problems with bindings
# Or give a better message.
#@+node:ekr.20100113075303.6270: *4* Easiest vim problems
#@+node:ekr.20100112051224.6229: *5* (vim) Binding Arrow keys (fixed?)
Binding arrow keys, with or without Shift, Ctrl, Alt, and their combinations, to
commands or @mode nodes have no effect.
#@+node:ekr.20100112051224.6239: *5* Displaying mode help
@nocolor-node

The "--> mode-help" command has the following issues related to the
display of the "Help" tab:

1. Key label always capitalized.

Vim commands are mapped to both lower-case and upper-case keys but always appear
mapped to upper-case keys within the "Help" tab.

2. Layout of tab's contents.

To improve readability and better support narrow tab cards, display the mode's
label without the "enter-" and "-mode" text and place the key label before the
mode label.

For example, the following entries would change from::
    enter-vi-delete-line-mode d
    enter-vi-delete-to-begin-of-word-mode b
to::
    d : vi-delete-line
    b : vi-delete-to-begin-of-word
#@+node:ekr.20100112051224.6225: *5* Repeat last cursor movement command
Support the ';' key: repeat the last "To character" or "Find character" command.
#@+node:ekr.20090629183608.8445: *5* Cursors (easiest)
Vi normally uses two different "current character" designators depending on the
current state.

Insert state:

In the Insert state, a vertical bar is placed between two characters to indicate
where the next key will be inserted. Leo's cursor is of this type.

Command state: 

In the Command state, vi expects that the cursor is highlighting a current
character and provides commands to enter the insert state or paste text either
before or after that current character. Leo's vi emulation currently does not
support a "current character" cursor. As a result, inserting and pasting before
or after is replaced by inserting or pasting "at" the current cursor location.
For example, the 'i' and 'a' command are both mapped to enter the insert state
at the current cursor location.
#@+node:ekr.20100112051224.6231: *5* Undo command (fixed?)
Using the "undo" command (key 'u') to undo a change to a node's headline text
only works correctly after another node has been selected. It appears that
changes made to a node's headline text are not recorded in Leo's change history
until the edited node has lost focus.
#@+node:ekr.20100113075303.6271: *4* Mode-oriented bindings
#@+node:ekr.20100112051224.6228: *5* Binding numeric keys (modes-oriented bindings)
Mapping a number to a command or an @mode node works but can not be used as it
prevents the number from being entered as text while in Vi's insert state.
#@+node:ekr.20100112051224.6230: *5* Binding 'bksp' key (mode-oriented bindings)
Binding 'bksp' key to back-char to move back a character in command mode
prevents 'bksp' from deleting characters in text edit mode.
#@+node:ekr.20080616110054.2: *4* Support vim dot command
The ability to repeat the last editing related command by pressing the period
key is not supported and there is no workaround in place.

Binding keys within nodes:

Some commands can be "easily" repeated by having the command's mode
bind itself to the period key.  This is not currently working.

Support commands requesting input:

Add companion commands that reuse input.  For example, a zap-to-
character-again command could exist which will reuse the key entered
in the last zap-to-character command.  With this support, the mode
that performs the initial command would assign the period key to a
companion mode that is identical to the initial mode but with the zap-
to-character command replaced by the zap-to-character-again command.

Commands requiring companion commands are:
  zap-to-character
  find-character
  backward-find-character
  (Any others?)

Notes:

- The copy of the character should be saved somewhere that does NOT affect the
  contents of the clipboard.

- The same or a separate storage location can be used for all commands to retain
  a copy of the character entered by the user. It doesn't matter since only the
  last command is assigned to the period key to be re-executed.
#@+node:ekr.20100113075303.6272: *4* Argument handling
#@+node:ekr.20100112051224.6222: *5* Commands requesting user input (hard?)
Commands requesting user input must be the last command executed within an @mode
node. This prevents the implementation of commands such as "yank to <character>"
that requires a "copy to clipboard" operation after the "find-character"
command.
#@+node:ekr.20100112051224.6226: *5* Range prefix to commands (k.getArgs)
The ability to specify a numeric range prefix is not supported. For example,
entering "3dd" will not delete the next three lines and "20G" will not move the
cursor to the 20th line in the file.

#@+node:ekr.20100112051224.6227: *5* Range prefix to objects (k.getArgs)
The ability to specify a numeric range prefix to an object is not supported. For
example, the "d2fx" command should Delete up to and including the 2nd Found "x"
character.
#@+node:ekr.20100112051224.6223: *4* Editing node headlines using @mode nodes
Commands modifying or selecting headline text do not work correctly within a
@mode node.

This eliminates accurate implementation of vi's delete/change/substitute/yank
object commands. As a workaround, the commands are currently written to only
select the text. The user must perform the subsequent delete, change,
substitute, and yank.
#@+node:ekr.20100112051224.6246: *4* Missing commands
#@+node:ekr.20100112051224.6232: *5* Toggle case command
Leo provides support for switching to upper or lower case but no method exists
to toggle between cases (used by Vi's "~" command).

#@+node:ekr.20100112051224.6233: *5* Replace current character command
Vi's "r" command allows user to replace the current character with the next
entered character.
#@+node:ekr.20100112051224.6234: *5* Move current line
Vi has a collection of "z<movement>" commands that will move the
current line to the top, middle, and bottom of the screen.  They are
not supported in Leo.
#@+node:ekr.20100112051224.6235: *5* Move buffer up/down
Vi maps keys to scroll the text up/down one line and by half the
number of visible lines.  Leo does not support this.

#@+node:ekr.20100112051224.6236: *5* Word-related commands
Vi supports two types of words in its commands:

1. Words that consist of only a subset of the character set and
2. words that consist of all characters except the space and tab characters.

Leo's always considers a word to consist of a subset of characters
although some word related commands include different characters
than others.
#@+node:ekr.20100112051224.6237: *5* Forward and backward by sentences
Leo's sentence related functions:

- do not stop at empty lines.
- do not skip periods within words.
- do not stop at sentences ending in non-periods.
- do not stop at the end or beginning of the buffer.

Note: see forwardSentenceHelper and backSentenceHelper functions.
#@+node:ekr.20100112051224.6238: *5* Focus to body pane
Leo functions exist which unconditionally set focus to the body pane
regardless of the active pane.

For example, bracket matching commands ("%" key) do not work within
a node's headline text.  Instead, the command is performed on the
node's body text.
#@+node:ekr.20090629183608.8446: *5* Notes about commands
Yank vs. Yank:
Vi's "yank" commands copy the selected text TO the clipboard.
Leo's "yank" commands insert text FROM the clipboard.

copy-text in modes:
Leo's copy-text command does not work within a mode.  As a result,
all "copy to clipboard" capability is being implemented using the
kill-<object> command followed by Leo's "yank" command to put the
text back.

paste-text in modes:
The paste-text command does not work within an @mode node.  Leo's
"yank" command is used instead.

delete-node does not copy node to clipboard:
A copy-node command is issued to copy the node to the clipboard
followed by the delete-node command.
#@+node:ekr.20100521090440.5890: *3* Cleanups: first
#@+node:ekr.20100223123910.5930: *4* recentFilesController
@
The present operation of recent files is surprising.

Recent files should be a global list, managed by a single controller.
#@+node:ekr.20100219083854.5615: *4* Improve caching
#@+node:ekr.20100209160132.5770: *5* cache notes
@nocolor-node

Top-level folder are direct subfolders of .leo/db.
Top-level folders represent file *locations* not file contents.
Exception: the top-level "globals" folder represents minor data.

Only two files are ever needed in a top-level folder:

contents_<key>: the contents of the file.
data_<key>: a dict representing the "minor data" of the file:
    <globals> element stuff, expansion bits, etc.

We write contents_<key> only once.
By definition, its contents never changes, since the contents generates the key.
We can write data_<key> as many times as we like.

To do:
- Simplify or even eliminate the path-manipulation code in PickleShareDB.
- Use g.makeAllNonExistentDirectories to make top-level directories.
- Clear cache should clear all top-level directories.
#@+node:ekr.20100209114432.5751: *6* Cache expansion bits
# Simplify the structure of the cache: put more into the "minor" files.
#@+node:ekr.20100211095442.6201: *5* cache notes 2
@nocolor-node

1. Memory does leak, and that's not ok with me.  And I want just two
files per top-level directory.

2. Strange things can happen with caching, as just happened to me when
I restored qtui_generate.py mistakenly deleted from leo/test.  There
is an @auto node for this file in qtGui.py, and I got improper 'can
not open' messages for this file.

3. It is troubling that the present caching scheme does not use the
full path to a file, only the basename.  This means that two identical
files in two different places will use the same cache entries.  I've
been wondering for the last several days about whether this could
cause problems.  I don't know for sure, but I am uncomfortable.

4. I want the clear-cache and clear-all-caches commands to do what
they say: get rid of everything.  Among other things, this is good for
debugging and recovering from cache problems.

#@+node:ekr.20100223075705.5635: *5* Don't write expansion bits
#@+node:ekr.20100210163813.5748: *5* Caching buglets?
@nocolor-node

This is a recent bug, but imo it has uncovered some other caching buglets. These
buglets are not big enough to delay Leo 4.7, but the new caching scheme would
ensure they never bite.

1. The code that computes what I have been calling the top-level directory is dubious::

    dbdirname = join(g.app.homeLeoDir,'db',
            '%s_%s' % (bname,hashlib.md5(fn).hexdigest()))

The problem is that bname is only the base name of the cached file, not a name
(or key) that depends on the full path. Thus, two copies of the same file in the
same place will be cached in the same directory. Is this ominous?

2. It's not clear what caching to do with the save-to command.
#@+node:ekr.20100225102636.5627: *5* Use the string returned by cacher
# It should be possible to avoid duplicate reads.
#@+node:ekr.20100211072044.5754: *4* Dubious usages of .unitTesting
#@+node:ekr.20031218072017.1723: *5* createMenuEntries
def createMenuEntries (self,menu,table,dynamicMenu=False):

    '''Create a menu entry from the table.
    New in 4.4: this method shows the shortcut in the menu,
    but this method **never** binds any shortcuts.'''

    c = self.c ; f = c.frame ; k = c.k
    if g.app.unitTesting: return

    trace = False
    for data in table:
        << get label & command or continue >>
        << compute commandName & accel from label & command >>
        # Bug fix: 2009/09/30: use canonical stroke.
        accelerator = k.shortcutFromSetting(accel,addKey=False) or ''
        stroke = k.shortcutFromSetting(accel,addKey=True) or ''
        if accelerator:
            accelerator = g.stripBrackets(k.prettyPrintKey(accelerator))
        if trace: # and commandName == 'add-comments':
            g.trace(bunch.val,repr(stroke),repr(accelerator),commandName)
        def masterMenuCallback (c=c,k=k,stroke=stroke,command=command,commandName=commandName,event=None):
            # if trace: g.trace('stroke',stroke)
            return k.masterMenuHandler(stroke,command,commandName)

        realLabel = self.getRealMenuName(label)
        amp_index = realLabel.find("&")
        realLabel = realLabel.replace("&","")
        if sys.platform == 'darwin':
            << clear accelerator if it is a plain key >>

        # c.add_command ensures that c.outerUpdate is called.
        if menu:
            c.add_command(menu,label=realLabel,
                accelerator=accelerator,
                command=masterMenuCallback,
                underline=amp_index)
#@+node:ekr.20051021091958: *6* << get label & command or continue >>
if g.isString(data):
    # New in Leo 4.4.2: Can use the same string for both the label and the command string.
    ok = True
    s = data
    removeHyphens = s and s[0]=='*'
    if removeHyphens: s = s[1:]
    label = self.capitalizeMinibufferMenuName(s,removeHyphens)
    command = s.replace('&','').lower()
    if label == '-':
        self.add_separator(menu)
        continue # That's all.
else:
    ok = type(data) in (type(()), type([])) and len(data) in (2,3)
    if ok:
        if len(data) == 2:
            # New in 4.4b2: command can be a minibuffer-command name (a string)
            label,command = data
        else:
            # New in 4.4: we ignore shortcuts bound in menu tables.
            label,junk,command = data

        if label in (None,'-'):
            self.add_separator(menu)
            continue # That's all.
    else:
        g.trace('bad data in menu table: %s' % repr(data))
        continue # Ignore bad data
#@+node:ekr.20031218072017.1725: *6* << compute commandName & accel from label & command >>
# New in 4.4b2: command can be a minibuffer-command name (a string)
minibufferCommand = type(command) == type('')
accel = None
if minibufferCommand:
    commandName = command 
    command = c.commandsDict.get(commandName)
    if command:
        rawKey,bunchList = c.config.getShortcut(commandName)
        # Pick the first entry that is not a mode.
        for bunch in bunchList:
            if not bunch.pane.endswith('-mode'):
                accel = bunch and bunch.val
                if bunch.pane  == 'text': break # New in Leo 4.4.2: prefer text bindings.
    else:
        if not g.app.unitTesting and not dynamicMenu:
            # Don't warn during unit testing.
            # This may come from a plugin that normally isn't enabled.
            if trace: g.trace('No inverse for %s' % commandName)
        continue # There is no way to make this menu entry.
else:
    # First, get the old-style name.
    commandName = self.computeOldStyleShortcutKey(label)
    rawKey,bunchList = c.config.getShortcut(commandName)
    for bunch in bunchList:
        if not bunch.pane.endswith('-mode'):
            if trace: g.trace('2','%20s' % (bunch.val),commandName)
            accel = bunch and bunch.val ; break
    # Second, get new-style name.
    if not accel:
        << compute emacs_name >>
            # Contains the not-so-horrible kludge.
        if emacs_name:
            commandName = emacs_name
            rawKey,bunchList = c.config.getShortcut(emacs_name)
            # Pick the first entry that is not a mode.
            for bunch in bunchList:
                if not bunch.pane.endswith('-mode'):
                    accel = bunch.val
                    if trace: g.trace('3','%20s' % (bunch.val),commandName)
                    break
        elif not dynamicMenu:
            g.trace('No inverse for %s' % commandName)
#@+node:ekr.20051021100806.1: *7* << compute emacs_name >>
@ One not-so-horrible kludge remains.

The cut/copy/paste commands in the menu tables are not the same as the methods
actually bound to cut/copy/paste-text minibuffer commands, so we must do a bit
of extra translation to discover whether the user has overridden their
bindings.
@c

if command in (f.OnCutFromMenu,f.OnCopyFromMenu,f.OnPasteFromMenu):
    emacs_name = '%s-text' % commandName
else:
    try: # User errors in the table can cause this.
        emacs_name = k.inverseCommandsDict.get(command.__name__)
    except Exception:
        emacs_name = None
#@+node:ekr.20060216110502: *6* << clear accelerator if it is a plain key >>
for z in ('Alt','Ctrl','Command'):
    if accelerator.find(z) != -1:
        break # Found.
else:
    accelerator = ''
#@+node:ekr.20051022043608.1: *5* createOpenWithMenuItemsFromTable
def createOpenWithMenuItemsFromTable (self,menu,table):

    '''Create an entry in the Open with Menu from the table.

    Each entry should be a sequence with 2 or 3 elements.'''

    c = self.c ; k = c.k

    if g.app.unitTesting: return

    for data in table:
        << get label, accelerator & command or continue >>
        # g.trace(label,accelerator)
        realLabel = self.getRealMenuName(label)
        underline=realLabel.find("&")
        realLabel = realLabel.replace("&","")
        callback = self.defineOpenWithMenuCallback(openWithData)

        c.add_command(menu,label=realLabel,
            accelerator=accelerator or '',
            command=callback,underline=underline)
#@+node:ekr.20051022043713.1: *6* << get label, accelerator & command or continue >>
ok = (
    type(data) in (type(()), type([])) and
    len(data) in (2,3)
)

if ok:
    if len(data) == 2:
        label,openWithData = data ; accelerator = None
    else:
        label,accelerator,openWithData = data
        accelerator = k.shortcutFromSetting(accelerator)
        accelerator = accelerator and g.stripBrackets(k.prettyPrintKey(accelerator))
else:
    g.trace('bad data in Open With table: %s' % repr(data))
    continue # Ignore bad data
#@+node:ekr.20031218072017.1725: *5* << compute commandName & accel from label & command >>
# New in 4.4b2: command can be a minibuffer-command name (a string)
minibufferCommand = type(command) == type('')
accel = None
if minibufferCommand:
    commandName = command 
    command = c.commandsDict.get(commandName)
    if command:
        rawKey,bunchList = c.config.getShortcut(commandName)
        # Pick the first entry that is not a mode.
        for bunch in bunchList:
            if not bunch.pane.endswith('-mode'):
                accel = bunch and bunch.val
                if bunch.pane  == 'text': break # New in Leo 4.4.2: prefer text bindings.
    else:
        if not g.app.unitTesting and not dynamicMenu:
            # Don't warn during unit testing.
            # This may come from a plugin that normally isn't enabled.
            if trace: g.trace('No inverse for %s' % commandName)
        continue # There is no way to make this menu entry.
else:
    # First, get the old-style name.
    commandName = self.computeOldStyleShortcutKey(label)
    rawKey,bunchList = c.config.getShortcut(commandName)
    for bunch in bunchList:
        if not bunch.pane.endswith('-mode'):
            if trace: g.trace('2','%20s' % (bunch.val),commandName)
            accel = bunch and bunch.val ; break
    # Second, get new-style name.
    if not accel:
        << compute emacs_name >>
            # Contains the not-so-horrible kludge.
        if emacs_name:
            commandName = emacs_name
            rawKey,bunchList = c.config.getShortcut(emacs_name)
            # Pick the first entry that is not a mode.
            for bunch in bunchList:
                if not bunch.pane.endswith('-mode'):
                    accel = bunch.val
                    if trace: g.trace('3','%20s' % (bunch.val),commandName)
                    break
        elif not dynamicMenu:
            g.trace('No inverse for %s' % commandName)
#@+node:ekr.20051021100806.1: *6* << compute emacs_name >>
@ One not-so-horrible kludge remains.

The cut/copy/paste commands in the menu tables are not the same as the methods
actually bound to cut/copy/paste-text minibuffer commands, so we must do a bit
of extra translation to discover whether the user has overridden their
bindings.
@c

if command in (f.OnCutFromMenu,f.OnCopyFromMenu,f.OnPasteFromMenu):
    emacs_name = '%s-text' % commandName
else:
    try: # User errors in the table can cause this.
        emacs_name = k.inverseCommandsDict.get(command.__name__)
    except Exception:
        emacs_name = None
#@+node:ekr.20061031131434.49: *5* scan
def scan (self,event=None,verbose=True,thread=True):

    c = self.c
    if not c or not c.exists or c.frame.isNullFrame: return
    if g.app.unitTesting: return

    # g.trace('autocompleter')

    if 0: # thread:
        # Use a thread to do the initial scan so as not to interfere with the user.            
        def scan ():
            #g.es("This is for testing if g.es blocks in a thread", color = 'pink' )
            # During unit testing c gets destroyed before the scan finishes.
            if not g.app.unitTesting:
                self.scanOutline(verbose=True)

        t = threading.Thread(target=scan)
        t.setDaemon(True)
        t.start()
    else:
        self.scanOutline(verbose=verbose)
#@+node:ekr.20060127052111.1: *5* cutStack
def cutStack (self):

    u = self ; n = u.max_undo_stack_size

    if n > 0 and u.bead >= n and not g.app.unitTesting:

        # Do nothing if we are in the middle of creating a group.
        i = len(u.beads)-1
        while i >= 0:
            bunch = u.beads[i]
            if hasattr(bunch,'kind') and bunch.kind == 'beforeGroup':
                return
            i -= 1

        # This work regardless of how many items appear after bead n.
        # g.trace('Cutting undo stack to %d entries' % (n))
        u.beads = u.beads[-n:]
        u.bead = n-1
        # g.trace('bead:',u.bead,'len(u.beads)',len(u.beads),g.callers())
#@+node:ekr.20061024075542.1: *5* open
def open (fileName=None):

    global g

    init()

    if g.app.unitTesting:
        return

    if not fileName:
        g.es_print('','leoPymacs.open:','no file name')
        return None

    # openWithFileName checks to see if the file is already open.
    ok, frame = g.openWithFileName(
        fileName,
        old_c=None,
        enableLog=False,
        readAtFileNodesFlag=True)

    c = ok and frame.c or None
    if c:
        g.es_print('','leoPymacs.open:',c)
    else:
        g.es_print('','leoPymacs.open:','can not open',fileName)

    return c
#@+node:ekr.20040715155607: *5* g.scanForAtIgnore
def scanForAtIgnore(c,p):

    """Scan position p and its ancestors looking for @ignore directives."""

    if g.app.unitTesting:
        return False # For unit tests.

    for p in p.self_and_parents():
        d = g.get_directives_dict(p)
        if 'ignore' in d:
            return True

    return False
#@+node:tbrown.20090219095555.61: *5* g.handleUrlInUrlNode
def handleUrlInUrlNode(url):

    # Note 1: the UNL plugin has its own notion of what a good url is.

    # Note 2: tree.OnIconDoubleClick now uses the body text of an @url
    #         node if it exists.

    if g.unitTesting: return
    << check the url; return if bad >>
    << pass the url to the web browser >>
#@+node:tbrown.20090219095555.62: *6* << check the url; return if bad >>
@ A valid url is (according to D.T.Hein):

3 or more lowercase alphas, followed by,
one ':', followed by,
one or more of: (excludes !"#;<>[\]^`|)
  $%&'()*+,-./0-9:=?@A-Z_a-z{}~
followed by one of: (same as above, except no minus sign or comma).
  $%&'()*+/0-9:=?@A-Z_a-z}~
@c

urlPattern = "[a-z]{3,}:[\$-:=?-Z_a-z{}~]+[\$-+\/-:=?-Z_a-z}~]"

if not url or len(url) == 0:
    g.es("no url following @url")
    return

# Add http:// if required.
if not re.match('^([a-z]{3,}:)',url):
    url = 'http://' + url
if not re.match(urlPattern,url):
    g.es("invalid url:",url)
    return
#@+node:tbrown.20090219095555.63: *6* << pass the url to the web browser >>
@ Most browsers should handle the following urls:
  ftp://ftp.uu.net/public/whatever.
  http://localhost/MySiteUnderDevelopment/index.html
  file://home/me/todolist.html
@c

try:
    import os
    os.chdir(g.app.loadDir)
    if g.match(url,0,"file:") and url[-4:]==".leo":
        ok,frame = g.openWithFileName(url[5:],None)
    else:
        import webbrowser
        # Mozilla throws a weird exception, then opens the file!
        try: webbrowser.open(url)
        except: pass
except:
    g.es("exception opening",url)
    g.es_exception()
#@+node:ekr.20050920084036.229: *5* yankRectangle
def yankRectangle (self,event,killRect=None):

    '''Yank into the rectangle defined by the start and end of selected text.'''

    c = self.c ; k = self.k
    w = self.editWidget(event)
    if not w: return

    killRect = killRect or self.theKillRectangle
    if g.app.unitTesting:
        # This value is used by the unit test.
        killRect = ['Y1Y','Y2Y','Y3Y','Y4Y']
    elif not killRect:
        k.setLabelGrey('No kill rect') ; return

    w,r1,r2,r3,r4 = self.beginCommand('yank-rectangle')

    n = 0
    for r in range(r1,r3+1):
        # g.trace(n,r,killRect[n])
        if n >= len(killRect): break
        w.delete('%s.%s' % (r,r2), '%s.%s' % (r,r4))
        w.insert('%s.%s' % (r,r2), killRect[n])
        n += 1

    i = '%s.%s' % (r1,r2)
    j = '%s.%s' % (r3,r2+len(killRect[n-1]))
    w.setSelectionRange(i,j,insert=j)

    self.endCommand()
#@+node:ekr.20050920084036.232: *5* stringRectangle
def stringRectangle (self,event):

    '''Prompt for a string, then replace the contents of a rectangle
    with a string on each line.'''

    c = self.c ; k = self.k ; state = k.getState('string-rect')
    if g.app.unitTesting:
        state = 1 ; k.arg = 's...s' # This string is known to the unit test.
        w = self.editWidget(event)
        self.stringRect = self.getRectanglePoints(w)
    if state == 0:
        w = self.editWidget(event) # sets self.w
        if not w or not self.check(event): return
        self.stringRect = self.getRectanglePoints(w)
        k.setLabelBlue('String rectangle: ',protect=True)
        k.getArg(event,'string-rect',1,self.stringRectangle)
    else:
        k.clearState()
        k.resetLabel()
        c.bodyWantsFocus()
        w = self.w
        self.beginCommand('string-rectangle')
        r1, r2, r3, r4 = self.stringRect
        s = w.getAllText()
        for r in range(r1,r3+1):
            i = g.convertRowColToPythonIndex(s,r-1,r2)
            j = g.convertRowColToPythonIndex(s,r-1,r4)
            s = s[:i] + k.arg + s[j:]
        w.setAllText(s)
        i = g.convertRowColToPythonIndex(s,r1-1,r2)
        j = g.convertRowColToPythonIndex(s,r3-1,r2+len(k.arg))
        w.setSelectionRange(i,j)
        self.endCommand()
        # 2010/1/1: Fix bug 480422:
        # string-rectangle kills syntax highlighting.
        c.frame.body.recolor(c.p,incremental=False)

#@+node:ekr.20051218133207: *5* backwardParagraphHelper
def backwardParagraphHelper (self,event,extend):

    w = self.editWidget(event)
    if not w: return

    s = w.getAllText()
    i,j = w.getSelectionRange()
    # A hack for wx gui: set the insertion point to the end of the selection range.
    if g.app.unitTesting:
        w.setInsertPoint(j)
    i,j = g.getLine(s,j)
    line = s[i:j]

    if line.strip():
        # Find the start of the present paragraph.
        while i > 0:
            i,j = g.getLine(s,i-1)
            line = s[i:j]
            if not line.strip(): break

    # Find the end of the previous paragraph.
    while i > 0:
        i,j = g.getLine(s,i-1)
        line = s[i:j]
        if line.strip():
            i = j-1 ; break

    self.moveToHelper(event,i,extend)
#@+node:ekr.20090103070824.11: *5* c.checkFileTimeStamp
def checkFileTimeStamp (self,fn):

    '''
    Return True if the file given by fn has not been changed
    since Leo read it or if the user agrees to overwrite it.
    '''

    trace = False and not g.unitTesting
    c = self

    # Don't assume the file still exists.
    if not g.os_path_exists(fn):
        if trace: g.trace('file no longer exists',fn)
        return True

    timeStamp = c.timeStampDict.get(fn)
    if not timeStamp:
        if trace: g.trace('no time stamp',fn)
        return True

    timeStamp2 = os.path.getmtime(fn)
    if timeStamp == timeStamp2:
        if trace: g.trace('time stamps match',fn,timeStamp)
        return True

    if g.app.unitTesting:
        return False

    if trace:
        g.trace('mismatch',timeStamp,timeStamp2)

    message = '%s\n%s\n%s' % (
        fn,
        g.tr('has been modified outside of Leo.'),
        g.tr('Overwrite this file?'))
    ok = g.app.gui.runAskYesNoCancelDialog(c,
        title = 'Overwrite modified file?',
        message = message)

    return ok == 'yes'
#@+node:ekr.20031218072017.2817: *5*  doCommand
command_count = 0

def doCommand (self,command,label,event=None):

    """Execute the given command, invoking hooks and catching exceptions.

    The code assumes that the "command1" hook has completely handled the command if
    g.doHook("command1") returns False.
    This provides a simple mechanism for overriding commands."""

    c = self ; p = c.p
    commandName = command and command.__name__
    c.setLog()

    self.command_count += 1
    if not g.app.unitTesting and c.config.getBool('trace_doCommand'):
        g.trace(commandName)

    # The presence of this message disables all commands.
    if c.disableCommandsMessage:
        g.es(c.disableCommandsMessage,color='blue')
        return 'break' # Inhibit all other handlers.

    if c.exists and c.inCommand and not g.unitTesting:
        # g.trace('inCommand',c)
        g.app.commandInterruptFlag = True
        g.es('ignoring command: already executing a command.',color='red')
        return 'break'

    g.app.commandInterruptFlag = False

    if label and event is None: # Do this only for legacy commands.
        if label == "cantredo": label = "redo"
        if label == "cantundo": label = "undo"
        g.app.commandName = label

    if not g.doHook("command1",c=c,p=p,v=p,label=label):
        try:
            c.inCommand = True
            val = command(event)
            if c and c.exists: # Be careful: the command could destroy c.
                c.inCommand = False
                c.k.funcReturn = val
            # else: g.pr('c no longer exists',c)
        except Exception:
            c.inCommand = False
            if g.app.unitTesting:
                raise
            else:
                g.es_print("exception executing command")
                g.es_exception(c=c)

        if c and c.exists:
            if c.requestCloseWindow:
                g.trace('Closing window after command')
                c.requestCloseWindow = False
                g.app.closeLeoWindow(c.frame)
            else:
                c.outerUpdate()

    # Be careful: the command could destroy c.
    if c and c.exists:
        p = c.p
        g.doHook("command2",c=c,p=p,v=p,label=label)

    return "break" # Inhibit all other handlers.
#@+node:ekr.20090712050729.6017: *5* promptForDangerousWrite
def promptForDangerousWrite (self,fileName,kind):

    c = self.c

    if g.app.unitTesting:
        val = g.app.unitTestDict.get('promptForDangerousWrite')
        return val in (None,True)

    # g.trace(timeStamp, timeStamp2)
    message = '%s %s\n%s\n%s' % (
        kind, fileName,
        g.tr('already exists.'),
        g.tr('Overwrite this file?'))

    ok = g.app.gui.runAskYesNoCancelDialog(c,
        title = 'Overwrite existing file?',
        message = message)

    return ok == 'yes'
#@+node:ekr.20070529083836: *5* cleanLines
def cleanLines (self,p,s):

    '''Return a copy of s, with all trailing whitespace removed.
    If a change was made, update p's body text and set c dirty.'''

    c = self.c ; cleanLines = [] ; changed = False
    lines = g.splitLines(s)
    for line in lines:
        if line.strip():
            cleanLines.append(line)
        elif line.endswith('\n'):
            cleanLines.append('\n')
            if line != '\n': changed = True
        else:
            cleanLines.append('')
            if line != '': changed = True
    s = g.joinLines(cleanLines)

    if changed and not g.app.unitTesting:
        p.setBodyString(s)
        c.setChanged(True)

    return s
#@+node:ekr.20080711093251.5: *5* at.writeOneAtShadowNode & helpers
def writeOneAtShadowNode(self,p,toString,force):

    '''Write p, an @shadow node.

    File indices *must* have already been assigned.'''

    trace = False and not g.unitTesting
    at = self ; c = at.c ; x = c.shadowController
    root = p.copy() 

    fn = p.atShadowFileNodeName()
    if trace: g.trace(p.h,fn)
    if not fn:
        g.es_print('can not happen: not an @shadow node',p.h,color='red')
        return False

    # A hack to support unknown extensions.
    self.adjustTargetLanguage(fn) # May set c.target_language.

    fn = at.fullPath(p)
    at.default_directory = g.os_path_dirname(fn)
    exists = g.os_path_exists(fn)
    if trace: g.trace('exists %s fn %s' % (exists,fn))

    # Bug fix 2010/01/18: Make sure we can compute the shadow directory.
    private_fn = x.shadowPathName(fn)
    if not private_fn:
        return False

    if not toString and not hasattr(root.v,'at_read') and exists:
        # Prompt if writing a new @shadow node would overwrite the existing public file.
        ok = self.promptForDangerousWrite(fn,kind='@shadow')
        if ok:
            root.v.at_read = True # Create the attribute for all clones.
        else:
            g.es("not written:",fn)
            return

    c.endEditing() # Capture the current headline.

    at.initWriteIvars(root,targetFileName=None, # Not used.
        atShadow=True,
        nosentinels=None, # set below.  Affects only error messages (sometimes).
        thinFile=True, # New in Leo 4.5 b2: private files are thin files.
        scriptWrite=False,
        toString=False, # True: create a fileLikeObject.  This is done below.
        forcePythonSentinels=True) # A hack to suppress an error message.
            # The actual sentinels will be set below.

    # Bug fix: Leo 4.5.1: use x.markerFromFileName to force the delim to match
    #                     what is used in x.propegate changes.
    marker = x.markerFromFileName(fn)
    at.startSentinelComment,at.endSentinelComment=marker.getDelims()

    if g.app.unitTesting: ivars_dict = g.getIvarsDict(at)

    # Write the public and private files to public_s and private_s strings.
    data = []
    for sentinels in (False,True):
        theFile = at.openStringFile(fn)
        at.sentinels = sentinels
        at.writeOpenFile(root,
            nosentinels=not sentinels,toString=False)
            # nosentinels only affects error messages, and then only if atAuto is True.
        s = at.closeStringFile(theFile)
        data.append(s)

    # Set these new ivars for unit tests.
    at.public_s, at.private_s = data

    if g.app.unitTesting:
        exceptions = ('public_s','private_s','sentinels','stringOutput')
        assert g.checkUnchangedIvars(at,ivars_dict,exceptions),'writeOneAtShadowNode'

    if at.errors == 0 and not toString:
        # Write the public and private files.
        if trace: g.trace('writing',fn)
        x.makeShadowDirectory(fn) # makeShadowDirectory takes a *public* file name.
        at.replaceFileWithString(private_fn,at.private_s)
        at.replaceFileWithString(fn,at.public_s)

    self.checkPythonCode(root,s=at.private_s,targetFn=fn)

    if at.errors == 0:
        root.clearOrphan()
        root.clearDirty()
    else:
        g.es("not written:",at.outputFileName,color='red')
        root.setDirty() # New in Leo 4.4.8.
        root.setOrphan() # 2010/10/22.

    return at.errors == 0
#@+node:ekr.20080819075811.13: *6* adjustTargetLanguage
def adjustTargetLanguage (self,fn):

    """Use the language implied by fn's extension if
    there is a conflict between it and c.target_language."""

    at = self ; c = at.c

    if c.target_language:
        junk,target_ext = g.os_path_splitext(fn)  
    else:
        target_ext = ''

    junk,ext = g.os_path_splitext(fn)

    if ext:
        if ext.startswith('.'): ext = ext[1:]

        language = g.app.extension_dict.get(ext)
        if language:
            c.target_language = language
        else:
            # An unknown language.
            pass # Use the default language, **not** 'unknown_language'
#@+node:bwmulder.20050101094804: *5* at.openForWrite
def openForWrite (self, filename, wb='wb'):

    '''Open a file for writes, handling shadow files.'''

    trace = False and not g.unitTesting
    at = self ; c = at.c ; x = c.shadowController

    try:
        shadow_filename = x.shadowPathName(filename)
        self.writing_to_shadow_directory = os.path.exists(shadow_filename)
        open_file_name       = g.choose(self.writing_to_shadow_directory,shadow_filename,filename)
        self.shadow_filename = g.choose(self.writing_to_shadow_directory,shadow_filename,None)

        if self.writing_to_shadow_directory:
            if trace: g.trace(filename,shadow_filename)
            x.message('writing %s' % shadow_filename)
            return 'shadow',open(open_file_name,wb)
        else:
            ok = c.checkFileTimeStamp(at.targetFileName)
            return 'check',ok and open(open_file_name,wb)

    except IOError:
        if not g.app.unitTesting:
            g.es_print('openForWrite: exception opening file: %s' % (open_file_name),color='red')
            g.es_exception()
        return 'error',None
#@+node:ekr.20031218072017.2609: *5* app.closeLeoWindow
def closeLeoWindow (self,frame):

    """Attempt to close a Leo window.

    Return False if the user veto's the close."""

    c = frame.c

    # g.trace('frame',frame,g.callers(4))

    c.endEditing() # Commit any open edits.

    if c.promptingForClose:
        # There is already a dialog open asking what to do.
        return False

    g.app.config.writeRecentFilesFile(c)
        # Make sure .leoRecentFiles.txt is written.

    if c.changed:
        c.promptingForClose = True
        veto = frame.promptForSave()
        c.promptingForClose = False
        if veto: return False

    g.app.setLog(None) # no log until we reactive a window.

    g.doHook("close-frame",c=c)
        # This may remove frame from the window list.

    if frame in g.app.windowList:
        g.app.destroyWindow(frame)

    if g.app.windowList:
        # Pick a window to activate so we can set the log.
        frame = g.app.windowList[0]
        frame.deiconify()
        frame.lift()
        frame.c.setLog()
        frame.c.bodyWantsFocus()
        frame.c.outerUpdate()
    elif not g.app.unitTesting:
        g.app.finishQuit()

    return True # The window has been closed.
#@+node:ekr.20041113113140: *5* loadOnePlugin
def loadOnePlugin (moduleOrFileName,tag='open0',verbose=False):

    trace = False # and not g.unitTesting

    global loadedModules,loadingModuleNameStack

    # Prevent Leo from crashing if .leoID.txt does not exist.
    if g.app.config is None:
        print ('No g.app.config, making stub...')
        class StubConfig(g.nullObject):
            pass
        g.app.config = StubConfig()

    # Fixed reversion: do this after possibly creating stub config class.
    verbose = False or verbose or g.app.config.getBool(c=None,setting='trace_plugins')
    warn_on_failure = g.app.config.getBool(c=None,setting='warn_when_plugins_fail_to_load')

    if moduleOrFileName.startswith('@'):
        if trace: g.trace('ignoring Leo directive')
        return False # Allow Leo directives in @enabled-plugins nodes.

    if moduleOrFileName.endswith('.py'):
        moduleName = 'leo.plugins.' + moduleOrFileName [:-3]
    elif moduleOrFileName.startswith('leo.plugins.'):
        moduleName = moduleOrFileName
    else:
        moduleName = 'leo.plugins.' + moduleOrFileName

    if isLoaded(moduleName):
        module = loadedModules.get(moduleName)
        if trace or verbose:
            g.trace('plugin',moduleName,'already loaded',color="blue")
        return module

    assert g.app.loadDir

    moduleName = g.toUnicode(moduleName)

    # This import will typically result in calls to registerHandler.
    # if the plugin does _not_ use the init top-level function.
    loadingModuleNameStack.append(moduleName)

    try:
        toplevel = __import__(moduleName)
        # need to look up through sys.modules, __import__ returns toplevel package
        result = sys.modules[moduleName]

    except g.UiTypeException:
        if not g.unitTesting and not g.app.batchMode:
            g.es_print('Plugin %s does not support %s gui' % (
                moduleName,g.app.gui.guiName()))
        result = None

    except ImportError:
        if trace or tag == 'open0': # Just give the warning once.
            g.es_print('error importing plugin:',moduleName,color='red')
            g.es_exception()
        result = None

    except Exception as e:
        g.es_print('exception importing plugin ' + moduleName,color='red')
        g.es_exception()
        result = None

    loadingModuleNameStack.pop()

    if result:
        loadingModuleNameStack.append(moduleName)

        if tag == 'unit-test-load':
            pass # Keep the result, but do no more.
        elif hasattr(result,'init'):
            try:
                # Indicate success only if init_result is True.
                init_result = result.init()
                # g.trace('result',result,'init_result',init_result)
                if init_result:
                    loadedModules[moduleName] = result
                    loadedModulesFilesDict[moduleName] = g.app.config.enabledPluginsFileName
                else:
                    if verbose and not g.app.initing:
                        g.es_print('loadOnePlugin: failed to load module',moduleName,color="red")
                    result = None
            except Exception:
                g.es_print('exception loading plugin',color='red')
                g.es_exception()
                result = None
        else:
            # No top-level init function.
            # Guess that the module was loaded correctly,
            # but do *not* load the plugin if we are unit testing.

            if g.app.unitTesting:
                result = None
                loadedModules[moduleName] = None
            else:
                g.trace('no init()',moduleName)
                loadedModules[moduleName] = result
        loadingModuleNameStack.pop()

    if g.app.batchMode or g.app.inBridge: # or g.unitTesting
        pass
    elif result:
        if trace or verbose:
            g.trace('loaded plugin:',moduleName,color="blue")
    else:
        if trace or warn_on_failure or (verbose and not g.app.initing):
            if trace or tag == 'open0':
                g.trace('can not load enabled plugin:',moduleName,color="red")

    return result
#@+node:ekr.20031218072017.2829: *5* c.openTempFileInExternalEditor
def openTempFileInExternalEditor(self,arg,fn,openType,testing=False):

    '''Open the closed mkstemp file fn in an external editor.
    The arg and openType args come from the data arg to c.openWith.
    '''

    trace = False and not g.unitTesting
    testing = testing or g.unitTesting
    if arg is None: arg = ''

    try:
        if trace: g.trace(repr(openType),repr(arg),repr(fn))
        command = '<no command>'
        if openType == 'os.system':
            if 1:
                # This works, *provided* that arg does not contain blanks.  Sheesh.
                command = 'os.system(%s)' % (arg+fn)
                if trace: g.trace(command)
                if not testing: os.system(arg+fn)
            else:
                # XP does not like this format!
                command = 'os.system("%s %s")' % (arg,fn)
                if not testing: os.system('"%s" "%s"' % (arg,fn))
        elif openType == 'os.startfile':
            command = 'os.startfile(%s)' % (arg+fn)
            if trace: g.trace(command)
            if not testing: os.startfile(arg+fn)
        elif openType == 'exec':
            command = 'exec(%s)' % (arg+fn)
            if trace: g.trace(command)
            if not testing: exec(arg+fn,{},{})
        elif openType == 'os.spawnl':
            filename = g.os_path_basename(arg)
            command = 'os.spawnl(%s,%s,%s)' % (arg,filename,fn)
            if trace: g.trace(command)
            if not testing: os.spawnl(os.P_NOWAIT,arg,filename,fn)
        elif openType == 'os.spawnv':
            filename = os.path.basename(arg[0]) 
            vtuple = arg[1:]
            vtuple.insert(0, filename)
                # add the name of the program as the first argument.
                # Change suggested by Jim Sizelove.
            vtuple.append(fn)
            command = 'os.spawnv(%s,%s)' % (arg[0],repr(vtuple))
            if trace: g.trace(command)
            if not testing: os.spawnv(os.P_NOWAIT,arg[0],vtuple)
        elif openType == 'subprocess.Popen':
            use_shell = True
            if g.isString(arg):
                if arg:
                    vtuple = arg + ' ' + fn
                else:
                    vtuple = fn
            elif isinstance(arg,(list, tuple)):
                vtuple = arg[:]
                vtuple.append(fn)
                use_shell = False
            command = 'subprocess.Popen(%s)' % repr(vtuple)
            if trace: g.trace(command)
            if not testing:
                try:
                    subprocess.Popen(vtuple,shell=use_shell)
                except OSError:
                    g.es_print('vtuple',repr(vtuple))
                    g.es_exception()
        elif g.isCallable(openType):
            # Invoke openWith like this:
            # c.openWith(data=[f,None,None])
            # f will be called with one arg, the filename
            if trace: g.trace('%s(%s)' % (openType,fn))
            command = '%s(%s)' % (openType,fn)
            if not testing: openType(fn)
        else:
            command='bad command:'+str(openType)
            if not testing: g.trace(command)
        return command # for unit testing.
    except Exception:
        g.es('exception executing open-with command:',command)
        g.es_exception()
        return 'oops: %s' % command
#@+node:ekr.20031218072017.1151: *5* tangle.put_all_roots
@
This is the top level method of the second pass. It creates a separate derived file
for each @root directive in the outline. The file is actually written only if
the new version of the file is different from the old version,or if the file did
not exist previously. If changed_only_flag FLAG is True only changed roots are
actually written.
@c

def put_all_roots(self):

    c = self.c ; outline_name = c.mFileName

    for section in self.root_list:

        # g.trace(section.name)
        file_name = c.os_path_finalize_join(self.tangle_directory,section.name)
        mode = c.config.output_newline
        textMode = mode == 'platform'
        if g.unitTesting:
            self.output_file = g.fileLikeObject()
            temp_name = 'temp-file'
        else:
            self.output_file,temp_name = g.create_temp_file(textMode=textMode)
        if not temp_name:
            g.es("can not create temp file")
            break
        <<Get root specific attributes>>
        <<Put @first lines>>
        if self.use_header_flag and self.print_mode == "verbose":
            << Write a banner at the start of the output file >>
        for part in section.parts:
            if part.is_root:
                self.tangle_indent = 0 # Initialize global.
                self.put_part_node(part,False) # output first lws
        self.onl() # Make sure the file ends with a cr/lf
        << unit testing fake files>>
        self.output_file.close()
        self.output_file = None
        << unit testing set result and continue >>
        if self.errors + g.app.scanErrors == 0:
            g.update_file_if_changed(c,file_name,temp_name)
        else:
            g.es("unchanged:",file_name)
            << Erase the temporary file >>
#@+node:ekr.20031218072017.1152: *6* <<Get root specific attributes>>
# Stephen Schaefer, 9/2/02
# Retrieve the full complement of state for the root node
self.language = section.root_attributes.language
self.use_header_flag = section.root_attributes.use_header_flag
self.print_mode = section.root_attributes.print_mode
self.path = section.root_attributes.path
self.page_width = section.root_attributes.page_width
self.tab_width = section.root_attributes.tab_width
# Stephen P. Schaefer, 9/13/2002
self.first_lines = section.root_attributes.first_lines
#@+node:ekr.20031218072017.1153: *6* <<Put @first lines>>
# Stephen P. Schaefer 9/13/2002
if self.first_lines:
    self.os(self.first_lines)
#@+node:ekr.20031218072017.1154: *6* <<Write a banner at the start of the output file>>
# a root section must have at least one part
assert len(section.parts)>0
delims=section.parts[0].delims
if delims[0]:
    self.os(delims[0])
    self.os(" Created by Leo from: ")
    self.os(outline_name)
    self.onl() ; self.onl()
elif delims[1] and delims[2]:
    self.os(delims[1])
    self.os(" Created by Leo from: ")
    self.os(outline_name)
    self.oblank() ; self.os(delims[2])
    self.onl() ; self.onl()
#@+node:ekr.20031218072017.1155: *6* << Erase the temporary file >>
try: # Just delete the temp file.
    os.remove(temp_name)
except: pass
#@+node:sps.20100608083657.20937: *6* << unit testing fake files>>
if g.unitTesting:
    # complications to handle testing of multiple @root directives together with
    # @path directives
    file_name_path = file_name
    if (file_name_path.find(c.openDirectory) == 0):
        relative_path = file_name_path[len(c.openDirectory):]
        # don't confuse /u and /usr as having common prefixes
        if (relative_path[:len(os.sep)] == os.sep):
            file_name_path = relative_path[len(os.sep):]
    self.tangle_output[file_name_path] = self.output_file.get()
#@+node:sps.20100608083657.20938: *6* << unit testing set result and continue >>
if g.unitTesting:
    assert self.errors == 0
    g.app.unitTestDict ['tangle'] = True
    g.app.unitTestDict ['tangle_directory'] = self.tangle_directory
    if g.app.unitTestDict.get('tangle_output_fn'):
        g.app.unitTestDict['tangle_output_fn'] += "\n" + file_name
    else:
        g.app.unitTestDict ['tangle_output_fn'] = file_name
    continue
#@+node:ekr.20041005105605.15: *5* at.initWriteIvars
def initWriteIvars(self,root,targetFileName,
    atAuto=False,atEdit=False,atShadow=False,
    nosentinels=False,thinFile=False,
    scriptWrite=False,toString=False,
    forcePythonSentinels=None,
):
    at = self ; c = at.c

    assert root
    self.initCommonIvars()

    at.atAuto = atAuto
    at.atEdit = atEdit
    at.atShadow = atShadow
    # at.default_directory: set by scanAllDirectives()
    at.docKind = None
    if forcePythonSentinels:
        at.endSentinelComment = None
    # else: at.endSentinelComment set by initCommonIvars.
    # at.encoding: set by scanAllDirectives() below.
    # at.explicitLineEnding # True: an @lineending directive specifies the ending.
        # Set by scanAllDirectives() below.
    at.fileChangedFlag = False # True: the file has actually been updated.
    at.force_newlines_in_at_nosent_bodies = c.config.getBool(
        'force_newlines_in_at_nosent_bodies')
    if forcePythonSentinels is None:
        forcePythonSentinels = scriptWrite
    # at.language:      set by scanAllDirectives() below.
    # at.outputFile:    set below.
    # at.outputNewline: set below.
    if forcePythonSentinels:
        # Force Python comment delims for g.getScript.
        at.startSentinelComment = "#"
    # else:                 set by initCommonIvars.
    # at.stringOutput:      set below.
    # at.outputFileName:    set below.
    # at.output_newline:    set by scanAllDirectives() below.
    # at.page_width:        set by scanAllDirectives() below.
    at.sentinels = not nosentinels
    at.shortFileName = ""   # For messages.
    at.root = root
    # at.tab_width:         set by scanAllDirectives() below.
    at.targetFileName = targetFileName
        # Must be None for @shadow.
    at.thinFile = thinFile
    at.toString = toString
    at.writeVersion5 = at.new_write and not atShadow

    at.scanAllDirectives(root,
        scripting=scriptWrite,
        forcePythonSentinels=forcePythonSentinels,
        issuePathWarning=True)
    # Sets the following ivars:
        # at.default_directory
        # at.encoding
        # at.explicitLineEnding
        # at.language
        # at.output_newline
        # at.page_width
        # at.tab_width

    if toString:
        at.outputFile = g.fileLikeObject()
        if g.app.unitTesting:
            at.output_newline = '\n'
        # else: at.output_newline set in initCommonIvars.
        at.stringOutput = ""
        at.outputFileName = "<string-file>"
    else:
        at.outputFile = None # The temporary output file.
        # at.outputNewline set in initCommonIvars.
        at.stringOutput = None
        at.outputFileName = g.u('')

    # Init all other ivars even if there is an error.
    if not at.errors and at.root:
        if hasattr(at.root.v,'tnodeList'):
            delattr(at.root.v,'tnodeList')
        at.root.v._p_changed = True
#@+node:ekr.20090131200406.11: *4* Remove remaining tk-isms from Leo's core
@nocolor-node

Eliminate all tk-indices from leoEditCommands.py

These are marked with ###

(found) wordend, wordstart
(found) lineend, linestart
(found) sel.first, sel.last
(found) w.insert, w.delete
#@+node:ekr.20100206074650.5844: *3* Features
#@+node:ekr.20090811141250.5955: *4*  Most important features
@ Important: There is another features list for 4.9.
#@+node:ekr.20100521130114.5904: *5* Document plugins automatically
@nocolor-node

We want better docs, automatically, and a way to update @enabled-plugins.

http://groups.google.com/group/leo-editor/browse_thread/thread/b409aac6ca8cd33b
#@+node:ekr.20090801103907.6018: *5* Add entries to global dicts for more languages
http://groups.google.com/group/leo-editor/browse_thread/thread/b41ddfeb3c84e780

Especially languages in leo/modes.
#@+node:ekr.20090816125009.5993: *6* @url http://groups.google.com/group/leo-editor/browse_thread/thread/b41ddfeb3c84e780
#@+node:ekr.20090814190307.5983: *6* print all modes/*.py files
import glob

theDir = g.os_path_finalize_join(g.app.loadDir,'..','modes','*.py')

aList = glob.glob(theDir)

for z in aList:
    print g.os_path_basename(z)

@
Exist:

    "ada"   : "ada",
    "adb"   : "ada",
    "ahk"   : "autohotkey",  # EKR: 2009-01-30.
    "as"    : "actionscript",
    "bas"   : "rapidq",
    "bat"   : "batch",
    "c"     : "c",
    "cfg"   : "config",
    "cpp"   : "cpp",
    "css"   : "css",
    "el"    : "elisp",
    "forth" : "forth",
    "f"     : "fortran",
    "f90"   : "fortran90",
    "h"     : "c",
    "html"  : "html",
    "ini"   : "ini",
    "java"  : "java",
    "ksh"   : "kshell", # Leo 4.5.1.
    "lua"   : "lua",  # ddm 13/02/06
    "nw"    : "noweb",
    "otl"   : "vimoutline",  #TL 8/25/08 Vim's outline plugin
    "p"     : "pascal",
    "pl"    : "perl",   # 11/7/05
    "pod"   : "perlpod", # 11/7/05
    "php"   : "php",
    "py"    : "python",
    "sql"   : "plsql", # qt02537 2005-05-27
    "r"     : "rebol",
    "rb"    : "ruby", # thyrsus 2008-11-05
    "rest"  : "rst",
    "rst"   : "rst",
    "sh"    : "shell",
    "tex"   : "tex",
    "txt"   : "plain",
    "tcl"   : "tcltk",
    "vim"   : "vim",
"w"     : "cweb",
"xml"   : "xml",
"xsl"   : "xslt",

Add first:


ada95.py
antlr.py
apacheconf.py
apdl.py
applescript.py
asp.py
aspect_j.py
assembly_macro32.py
assembly_mcs51.py
assembly_parrot.py
assembly_r2000.py
assembly_x86.py
awk.py
b.py
batch.py
bbj.py
bcel.py
bibtex.py
c.py
chill.py
cobol.py
coldfusion.py
cplusplus.py
csharp.py
css.py
cvs_commit.py
d.py
doxygen.py
dsssl.py
eiffel.py
embperl.py
erlang.py
factor.py
forth.py
fortran.py
fortran90.py
foxpro.py
freemarker.py
gettext.py
groovy.py
haskell.py
hex.py
html.py
i4gl.py
icon.py
idl.py
inform.py
ini.py
inno_setup.py
interlis.py
io.py
java.py
javascript.py
jcl.py
jhtml.py
jmk.py
jsp.py
latex.py
lilypond.py
lisp.py
lotos.py
lua.py
mail.py
makefile.py
maple.py
matlab.py
ml.py
modula3.py
moin.py
mqsc.py
netrexx.py
nqc.py
nsis2.py
objective_c.py
objectrexx.py
occam.py
omnimark.py
pascal.py
patch.py
perl.py
php.py
phpsection.py
pike.py
pl1.py
plain.py
plsql.py
pop11.py
postscript.py
povray.py
powerdynamo.py
progress.py
prolog.py
props.py
psp.py
ptl.py
pvwave.py
pyrex.py
python.py
r.py
rebol.py
redcode.py
relax_ng_compact.py
rest.py
rhtml.py
rib.py
rpmspec.py
rtf.py
ruby.py
rview.py
sas.py
scheme.py
sdl_pr.py
sgml.py
shell.py
shellscript.py
shtml.py
smalltalk.py
smi_mib.py
splus.py
sqr.py
squidconf.py
ssharp.py
svn_commit.py
swig.py
tcl.py
tex.py
texinfo.py
text.py
tpl.py
tsql.py
uscript.py
vbscript.py
velocity.py
verilog.py
vhdl.py
xml.py
xsl.py
zpt.py
__init__.py
#@+node:ekr.20090601083544.6051: *5* Add expand_noweb_references for rst3 plugin
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/3cd5cb06d32264d

> What would work for me is if named sections in a @rst subtree
> would work exactly as they work for other derived files: they
> get inserted at the place where they are referenced.

- Add support for the following options:
    - expand_noweb_references: default False for compatibility.
    - ignore_noweb definitions: default False for compatibility.

- When expand_noweb_references is True, definitions (typically clones)
  must be descendants of the referencing node (in the @rst tree)
#@+node:ekr.20090601083544.6052: *6* post
@nocolor-node

I want to write the documentation for the source program in a @rst3 subtree. In
this @rst3 subtree I want to refer to fragments of the program, like:

In the following code fragment::

  <<code fragment>>

Unfortunately, <<code fragment>> will not be expanded. Furthermore, in order to
get to this work, I should have <<code fragment>> under the @rst3 subtree as
well, but this is then also treated as @rst3 input (which in this case, is not
what I want).
#@+node:ekr.20080918164844.12: *5* Improve headline navigation
@nocolor

http://groups.google.com/group/leo-editor/browse_thread/thread/bce1065d9a332fcd

Now that the leo is more "modeless" (I'm speaking of switching between
outline navigation and body editing modes), which btw is a clear improvement
to how leo used to behave, here are some things that still feel a bit
un-intuitive:

- ctrl+H (edit-headline) "locks" the user into the mode way too much.

   * The headline editing mode should be cancelled when the user does:
      - cursor up/down
      - ESC

   * Even better alternative: I find myself constantly thinking that "ok,
now I need to edit a headline (typically not the current headline), so now
I'll press ctrl+H". I think perhaps pressing up/down should cancel the
current headline editing, and select the next/previous headline for editing.
That is, I wouldn't need to navigate to the headline I want to edit before I
start editing it.

  - cut-node (ctrl+shift+x) selects the wrong node after the cut. The
intuitive assumption is that cut will select the node that "took the place
of the
    current node", instead of starting to travel upwards the set of nodes.

   * Typical use case is the way you usually start deleting a set of items.
You move to the first item and start cutting repeatedly. This wont work with
the current behaviour. 
#@+node:ekr.20080730212711.7: *5* Improve plugins handling
@nocolor-node

- Don't repeat "can not load <plugin>" messages.

- Unify print-plugins and print-plugins-info commands, and make them more informative.

- Fix the plugin mess.

    - Create a way to unload a plugin.  This should be relatively easy.
#@+node:ekr.20090804105939.5993: *6* Possibly redo how plugins are loaded
#@+node:ekr.20080313032655.2: *6* Once a plugin is enabled, it is always enabled
#@+node:ekr.20080923200153.1: *6* Support scan-directives hook again?
# This affects the add_directives plugin.
# Also, the color_markup plugin doesn't work with the threading colorizer.
#@+node:ekr.20080421140032.1: *6* Fix multiple controllers problem in all plugins
@nocolor

http://groups.google.com/group/leo-editor/browse_thread/thread/663e1f9d8e2d1c24

@color
#@+node:ekr.20090724081340.5987: *5* Improve recursive import script and @auto
@nocolor-node

Instead of adding an @ignore directive, it might be better
to change @auto to @@auto.

Should @auto be more lenient with C files?

Improve the recursive import script.
    - Minimize the path names
    - Option to include/exclude the @auto itself

#@+node:ekr.20090907080624.6081: *5* Spell checker should check headlines
#@+node:ekr.20071001052501: *5* Versioning for nodes
@nocolor

One feature I have not seen in SCS system is something which might be called
"history compression": I might be interested in having both version 5 and 6
in my source tree, when the current version is 7, but I am not really interested
in the 2000 steps which transformed 5 into 6 (just suggested this feature to
the bazaar people). This happens actually quite often to me, since I use the
SCS as a back-up system, saving many (uninteresting) intermediate steps while
implementing a new feature.
#@+node:ekr.20101004092958.5914: *5* Write treepad scanner
@ treepad.py is from the treepad website
#@+node:ekr.20101004092958.5939: *6* treepad.py
@first #! /usr/local/bin/python

# treepad.py

@language python
@tabwidth -4
@others
if __name__ == '__main__':
    Main().Run()

#@+node:ekr.20101004092958.5940: *7* treepad declarations
import sys, os, re, string

# constants
VERSION = "<Treepad version 2.7>"

# regexes
END_RE = re.compile(r'^<end node> ([^ ]+)$')
#@+node:ekr.20101004092958.5941: *7* class Node
class Node:
    @others
#@+node:ekr.20101004092958.5942: *8* __init__
def __init__(self):
    self.title    = ""
    self.level    = 0
    self.article  = []
    self.children = []
    self.parent   = None
    self.end      = ""
#@+node:ekr.20101004092958.5943: *8* __str__
def __str__(self):
    return "%s/%d" % (self.title, self.level)
#@+node:ekr.20101004092958.5944: *8* addchild
def addchild(self, node):
    assert self.level == node.level-1 and node.parent is None
    node.parent = self
    self.children.append( node )
#@+node:ekr.20101004092958.5945: *8* findparent
def findparent(self, node):
    if self.level == (node.level-1): return self
    return self.parent.findparent(node)
#@+node:ekr.20101004092958.5946: *8* writenode
def writenode(self, fp):
    fp.write("dt=Text\n")
    fp.write("<node>\n")
    fp.write("%s\n" % self.title)
    fp.write("%s\n" % self.level)
    for line in self.article:
        fp.write("%s\n" % line)
    fp.write("<end node> %s\n" % self.end)
#@+node:ekr.20101004092958.5947: *8* writetree
def writetree(self, fp):
    self.writenode(fp)
    for node in self.children:
        node.writetree(fp)

#@+node:ekr.20101004092958.5948: *7* class NodeReader
class NodeReader:
    @others
#@+node:ekr.20101004092958.5949: *8* __init__
def __init__(self, fname, fp):
    self.fname    = fname
    self.fp       = fp
#@+node:ekr.20101004092958.5950: *8* expect
def expect(self, text, line=None):
    if line is None:
        line = self.fp.readline().strip()
    assert line == text, "expected " + line + " == " + text
#@+node:ekr.20101004092958.5951: *8* readstart
def readstart(self):
    self.expect(VERSION)
#@+node:ekr.20101004092958.5952: *8* readnode
def readnode(self):
    line = self.fp.readline()
    if line is None:
        return None
    line = line.strip()
    if len(line) < 1:
        return None
    self.expect("dt=Text", line)
    self.expect("<node>")
    node = Node()
    node.title = self.fp.readline().strip()
    node.level = int(self.fp.readline().strip())
    while 1:
        line = self.fp.readline()
        m = re.match(END_RE, line)
        if m:
            node.end = m.group(1).strip()
            break
        node.article.append( line.strip() )
    return node

#@+node:ekr.20101004092958.5953: *7* class TreeReader
class TreeReader:
    @others
#@+node:ekr.20101004092958.5954: *8* __init__
def __init__(self, fname, fp=None):
    if fp is None: fp = open(fname, 'r')
    self.nodereader = NodeReader(fname, fp)
    self.root = None
    self.prev = None
#@+node:ekr.20101004092958.5955: *8* add
def add(self, node):
    if self.prev is None:
        assert node.level == 0
        self.root = node
    else:
        assert node.level > 0
        parent = self.prev.findparent(node)
        parent.addchild( node )
    self.prev = node
#@+node:ekr.20101004092958.5956: *8* read
def read(self):
    self.nodereader.readstart()
    prev = None
    while 1:
        node = self.nodereader.readnode()
        if node is None: break
        self.add(node)

#@+node:ekr.20101004092958.5957: *7* class TreeWriter
class TreeWriter:
    @others
#@+node:ekr.20101004092958.5958: *8* __init__
def __init__(self, fname, fp=None):
    if fp is None: fp = open(fname, 'w')
    self.fname = fname
    self.fp    = fp
#@+node:ekr.20101004092958.5959: *8* write
def write(self, root):
    self.fp.write("%s\n" % VERSION)
    root.writetree(self.fp)

#@+node:ekr.20101004092958.5960: *7* class Main
class Main:
    @others
#@+node:ekr.20101004092958.5961: *8* __init__
def __init__(self):
    self.infile  = sys.argv[1]
    self.outfile = sys.argv[2]
    self.reader  = TreeReader(self.infile)
    self.writer  = TreeWriter(self.outfile)
#@+node:ekr.20101004092958.5962: *8* Run

def Run(self):
    self.reader.read()
    self.writer.write(self.reader.root)

#@+node:ekr.20090714085914.5994: *4* New commands
#@+node:ekr.20071004120359.2: *5* Do expand-region-abbrevs from
See: regionalExpandAbbrev.
#@+node:ekr.20090402072059.2: *5* clone-find-all-once
@

First do a normal clone-find-all for the word "clone". Then click the script
button and do it again. Notice that children of previously found nodes don't get
added again in the modified version.
#@+node:ekr.20090402072059.4: *6* @@@button my clone find all
import leo.core.leoFind as leoFind 

def new_find_all(self):
    << do some initial stuff >>


    while 1:
        pos, newpos = self.findNextMatch()
        if pos is None: break

        if count % 10 == 0 and count > 0:
            g.es("still searching, matches found: ", count)

        << Skip node if it's a child of a previously found node >>

        count += 1

        s = w.getAllText()
        i,j = g.getLine(s,pos)
        line = s[i:j]
        if not self.clone_find_all:
            self.printLine(line,allFlag=True)
        if self.clone_find_all and self.p.v.t not in clones:
            # g.trace(self.p.v.t,self.p.h)
            if not clones:
                undoData = u.beforeInsertNode(c.p)
                << create the found node >>
            clones.append(self.p.v.t)
            positions.append(self.p)
            << create a clone of p under the find node >>

    if self.clone_find_all and clones:
        u.afterInsertNode(found,undoType,undoData,dirtyVnodeList=[])
        c.selectPosition(found)
        c.setChanged(True)

    self.restore(data)
    c.redraw()
    g.es("found",count,"matches")


leoFind.leoFind.findAll = new_find_all

c.executeMinibufferCommand("clone-find-all") 
#@+node:ekr.20090402072059.5: *7* << do some initial stuff >>
g.es("findAll..., self: ", self)

c = self.c ; w = self.s_ctrl ; u = c.undoer
undoType = 'Clone Find All'
if not self.checkArgs():
    return
self.initInHeadline()
if self.clone_find_all:
    self.p = None # Restore will select the root position.
data = self.save()
self.initBatchCommands()
count = 0 ; clones = []; positions = []
#@+node:ekr.20090402072059.6: *7* << create the found node >>
oldRoot = c.rootPosition()
found = oldRoot.insertAfter()
found.moveToRoot(oldRoot)
c.setHeadString(found,'Found: ' + self.find_text)
c.setRootPosition(found) # New in Leo 4.5.
#@+node:ekr.20090402072059.7: *7* << create a clone of p under the find node >>
q = self.p.clone()
q.moveToLastChildOf(found)
#@+node:ekr.20090402072059.8: *7* << Skip node if it's a child of a previously found node >>
<< def: is a child of b >>

is_child_of_previous = False
for previously_found in positions:
    if is_a_child_of_b(self.p, previously_found):
        is_child_of_previous = True
        break

if is_child_of_previous:
    continue
#@+node:ekr.20090402072059.9: *8* << def: is a child of b >>

def is_a_child_of_b(a, b):
    for child in b.children_iter():
        if a.t == child.t:
            return True
        if is_a_child_of_b(a, child):
            return True
    return False
#@+node:ekr.20100108101415.6196: *5* insert-tab commands
insert-tab-or-indent-region, insert-hard-tab, insert-soft-tab.
#@+node:ekr.20090601083544.6067: *5* Make all commands available to all commanders
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/a83082cec5ad3df8

QQQ (Ville)
So to create command you would do

import leoPlugins

def mycmd(c,p, event):
  print c,p

leoPlugins.expose_command('my-command', mycmd)
leoPlugins.expose_button('press-this',mycmd)

[EKR: I would rather use g.app.exposeCommand, g.app.exposeButton].

These would be available on *all* new commanders, and the current
commander as well (for easy testing). Plugins would not have to
contain more code than what is presented above.

The idea is to have a global command dict, [EKR: g.app.commandsDict]
and global button dict.
There is one after-create-frame handler that introduces all the
entries in this dict to the commander command dict.
QQQ

============


QQQ
g.app.global_commands_dict, which gets copied to c on commander
creation. it's rev 1889, you'll note that it's a simple
implementation. I didn't add g.button yet. An example:

@g.command('bookmark')
def bookmark(event):
    c = event.get('c')
    p = c.currentPosition()
    bookmarks.append(p.gnx)
    g.es('bookmarked') 
QQQ
#@+node:ekr.20090131200406.15: *4* Optional file features
#@+node:ekr.20100827095120.5861: *5* Improve format of .leo files?
http://groups.google.com/group/leo-editor/browse_thread/thread/be5052957b21ad97

Maybe for Leo 4.9.
#@+node:ekr.20090622020908.6058: *5* Add lite sentinels
http://groups.google.com/group/leo-editor/browse_thread/thread/c4f2cf250600e4a9
#@+node:ekr.20080311135649.2: *5* Allow different .leo formats
@nocolor

On Tue, Mar 11, 2008 at 7:03 AM, Kent Tenney <kten...@gmail.com> wrote:

> On 3/11/08, derwisch <johannes.hues...@med.uni-heidelberg.de> wrote:

> >  On 11 Mrz., 08:03, "Ville M. Vainio" <vivai...@gmail.com> wrote:
> >  > It could also be argued that

> >  > - Referring to previous cloned vnodes explicitly in XML does not
> >  > necessarily obscure DAG - it follows the "do not repeat yourself"
> rule
> >  > - It will speed up reading
> >  > - Wouldn't it be better for preserving the integrity of the XML file?

> > I would lean towards this line of argumentation. A couple of days I
> >  had my Leo extension destroy the Leo ODM file (which was still valid
> >  according to Leo, but unreadable wrt the extension and broken uAs). I
> >  resorted to editing the Leo file with Emacs, and was quite surprised
> >  to see that the headStrings were attributes of vnodes.

> I'll chime in with my pet peeve re: .leo file structure::

> I think that putting the headstrings on vnodes and body strings on tnodes
> obscures the informational content of the .leo file, and makes the .leo
> file
> format less attractive as a generalized solution to the problem of how to
> manage head/body pairs which live in a hierarchal structure.

> Thanks,
> Kent

> >  I think that
> >  editing the file might have been a bit easier if there had been no
> >  such redundancy. But this is more a feeling rather than a qualified
> >  opinion.

Thanks for all these comments.  I'll respond to them all here.

Clearly, we should be using a standard xml parser to read .leo files.

My present thoughts:

- I personally like human-readable headlines in <v> elements.

- I am open to putting headlines in <t> elements, as an indication that
tnodes do indeed contain headlines and body text.

- I am willing to consider only writing shared subtrees once.

Oh! (An Aha)  All these are preferences.  We can allow any combination of
these provided that headlines appear somewhere.

So that's clean.  This will happen in Leo 4.5. 
#@+node:ekr.20061002093442: *5* Add opml support to new,open, save commands
#@+node:ekr.20071003104917: *5* Make http://edreamleo.org/namespaces/leo-python-editor/1.1 Leo's official namespace
xmlns:leo="http://edreamleo.org/namespaces/leo-python-editor/1.1"
#@+node:ekr.20090218115025.3: *5* Why are attributes pickled by default?
@nocolor-node

http://groups.google.com/group/leo-editor/browse_thread/thread/326a221f4c698f7a

> On Wed, Feb 18, 2009 at 12:12 PM, Kent Tenney <ktenney@gmail.com> wrote:
>>
>> Currently, Leo pickles the value of unknown attributes unless
>> the name starts with 'str_'
>>
>> Running the following code in node 'UA'
>>
>> p = c.currentPosition()
>> p.v.u = {'hello':'world', 'str_hello':'world'}
>>
>> results in the following in the .leo file:
>>
>> <v t="ktenney.20090218114928.367" str_hello="world"
>> hello="5505776f726c6471002e"><vh>UA</vh></v>
>>
>> I think this is surprising, Python-centric and contrary to the
>> spirit of Leo as a flexible data management platform.
>
> I suppose your point is that you can't create an arbitrarily named attribute
> with a string value. Does that create a real problem?

It requires a translation layer, either to (un)munge the name or
(un)pickle. Real problem?

Let's say each time I think 'I can use UAs to store that' I change
my mind when I realize my values will be in a pickle. (I really don't
want to name all my attributes str_xxx)

> As far as being Python-centric, can you suggest any other way of converting
> arbitrary data to a text string?

How is it done in any other XML file?
I've not used XML for arbitrary data, but it probably can be done.

> Why would that way be better than pickle?

My suspicion is that UAs would be used more for
storing text and numbers (as seems common for XML files)
than Python data objects.

Does Leo use UAs to store pickles?

I'm sure pickling capability is great, but I'm not convinced
it should be the _default._

No big deal.
#@+node:ekr.20080626081829.2: *5* Allow headline comments for @nosent files
@nocolor

http://groups.google.com/group/leo-editor/browse_thread/thread/eb718b4c6d478ac0

I'm just getting started learning how to use Leo. Now, I'd like to use
it for some of my projects, but there's no chance that I can convert
everyone at work to using it, so putting sentinel-filled files in our
repository is out of the question. At the same time, my code looks
awfully bare without sentinels because the documentation ends up in
the section names, not the comments!

So, I was wondering if there's a convenient way to pull the section
names into a comment at the start of each section?

===============

Interesting question.  Am I correct in assuming you are using @nosent trees
to generate your files?  If so, it would be easy to add support for the
following options:

@bool write_section_comments_in_at_nosent_trees
@bool write_node_name_comments_in_at_nosent_trees

The first would write a sentinel consisting of only the section name;
the second would write a sentinel consisting only of the node's headline
(for nodes whose headline is not a section name).

These seem like they would be useful additions.  One can even imagine
corresponding Leo directives so that the comments could be turned on or off
within an @nosent tree.

What do you think?

=====================

> Interesting question.  Am I correct in assuming you are using @nosent trees
> to generate your files?  If so, it would be easy to add support for the
> following options:

> @bool write_section_comments_in_at_nosent_trees
> @bool write_node_name_comments_in_at_nosent_trees

> The first would write a sentinel consisting of only the section name;
> the second would write a sentinel consisting only of the node's headline
> (for nodes whose headline is not a section name).

> These seem like they would be useful additions.  One can even imagine
> corresponding Leo directives so that the comments could be turned on or off
> within an @nosent tree.

That sounds like an excellent solution. Particularly the last bit --
if you could turn section-comments on and off as required, it would
become very convenient to use Leo to produce source that is intended
to also be read by non Leo users. 
#@+node:ekr.20080919085541.3: *5* Use sqlite data base as an alternative representation for .leo files
http://groups.google.com/group/leo-editor/browse_thread/thread/dff0c165e2211691
#@+node:ekr.20090210093316.1: *4* Optional features
#@+node:ekr.20081119132758.2: *5* Support @ifgui in settings trees
#@+node:ekr.20080727122007.1: *5* Allow user to set background colors of nodes
What uA should be used to specify node colors?

if the foreground / background color API uses uAs,
would/should the uAs use the reserved "leo_&lt;something&gt;" namespace?
#@+node:ekr.20080813064908.8: *5* Find a way to limit length of lines in indented nodes
#@+node:ekr.20080802070659.11: *5* Make node attributes visible, and supported by Leo's core
#@+node:ekr.20080806054207.3: *5* Auto scroll outline pane if headline would become invisible
@nocolor

http://groups.google.com/group/leo-editor/browse_thread/thread/76789df8aac08c70

When using leo as outliner, I often use only node headlines to write
down some data. If the headline string is too long, the cursor goes
beyond the visible area. When modifying a node headline, is it
possible to make leo to auto-scroll, so the cursor is always visible? 
#@+node:ekr.20070521105645: *5* Improve api docs with epydoc?
@nocolor
http://sourceforge.net/forum/message.php?msg_id=4319363
By: ktenney

I think there is room for improvement in documenting Leo's
API, making it easier to write these kind of scripts.
I'm not sure of the best way to do that.

Epydoc seems to be the most active project in this realm.
http://epydoc.sourceforge.net/epydoc.html
#@+node:ekr.20061116054917.6: *5* Remove blanks in calltips
#@+node:ekr.20080628095358.1: *5* Make each Leo command a class
http://groups.google.com/group/leo-editor/browse_thread/thread/5688ed9aaa39be2e#

@nocolor

The main difficulty I see in the migration is creating the tables in the getPublicCommands methods in the various classes in leoEditCommands.py.  At present, these tables associate command names (strings) with corresponding methods.  The form of getPublicCommands is always:

def getPublicCommands (self):
  return {
    'command-name1': self.methodName1,
    'command-name2': self.methodName2,
    ...
  }

Thinking out loud, let's see whether the migration can be done easily.  We would change the entry:

    'command-name1': self.methodNameN,

to:

    'command-name1': self.classNameN(self),

That is, the table creates an instance of the class by calling the class's ctor, with self (the container object) as the ctor's only argument.  To make this work, all we need to do is give the class a __call__ method whose signature matches the signature of methodNameN, that is, the signature used to call methods previously.

Well, isn't this nice.  We can transition gradually, as needed.  No need *ever* to do a mass migration.  It should be easy to verify this scheme with one or two examples.  Please report your experiences if you decide to play around with this.

Edward

P.S.  I think it would be good style to append "Class" to the name of each command class. This makes it clear that self.myCommandClass(self) is a ctor.
#@-all
#@-leo
