#!python
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License
# Version 1.1 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"
# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
# License for the specific language governing rights and limitations
# under the License.
#
# The Original Code is Komodo code.
#
# The Initial Developer of the Original Code is ActiveState Software Inc.
# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007
# ActiveState Software Inc. All Rights Reserved.
#
# Contributor(s):
#   ActiveState Software Inc
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

"""The API catalogs-zone of the codeintel database.
See the database/database.py module docstring for an overview.
"""

import sys
import os
from os.path import (join, dirname, exists, expanduser, splitext, basename,
                     split, abspath, isabs, isdir, isfile, normpath,
                     normcase)
import pickle as pickle
import threading
import time
from hashlib import md5
import bisect
import fnmatch
from glob import glob
from pprint import pprint, pformat
import logging
from io import StringIO
import codecs
import copy
import weakref
import queue

import ciElementTree as ET
from codeintel2.common import *
from codeintel2.buffer import Buffer
from codeintel2.util import dedent, safe_lang_from_lang, banner, hotshotit
from codeintel2.tree import tree_from_cix_path
from codeintel2.database.util import filter_blobnames_for_prefix
from codeintel2.database.resource import AreaResource


#---- globals
log = logging.getLogger("codeintel.db")
# log.setLevel(logging.DEBUG)


#---- Database zone and lib implementations
class CatalogsZone(object):
    """Singleton zone managing the db/catalogs/... area.

    TODO: Locking: .cull_mem() and .save() will be called periodically
          on indexer thread. Anything they access needs to be guarded.
    """
    _res_index = None
    _blob_index = None
    _toplevelname_index = None
    _toplevelprefix_index = None

    _have_updated_at_least_once = False

    def __init__(self, mgr, catalog_dirs=None):
        self.mgr = mgr
        self.db = mgr.db

        if catalog_dirs is None:
            catalog_dirs = []
        assert isinstance(catalog_dirs, list)
        self.catalog_dirs = catalog_dirs

        self.base_dir = join(self.db.base_dir, "db", "catalogs")

        self._lib_cache = {}  # (lang, selection_res_ids) -> CatalogLib

        self._lock = threading.RLock()
        self._blob_and_atime_from_blobname_from_lang_cache = {}
        self._dbsubpaths_and_lpaths_to_save = []

    def __repr__(self):
        return "<catalog zone>"

    def _selection_from_selector(self, selections):
        """Given a sequence of catalog selection strings (each is a
        catalog name or full path to a catalog .cix file) return a dict
        mapping:

            <normalized-selector> -> <selection-string>

        If "selections" is None, this returns None.
        """
        if selections is None:
            return None
        selection_from_selector = {}
        for selection in selections:
            if isabs(selection):
                selector = normpath(normcase(selection))
            else:
                selector = selection.lower()
            selection_from_selector[selector] = selection
        return selection_from_selector

    _res_ids_from_selector_cache = None

    def _res_ids_from_selections(self, selections):
        """Returns a tuple of the database resource ids for the given
        selections and a list of selections that didn't match any loaded
        resources.
        """
        if self._res_ids_from_selector_cache is None:
            cache = self._res_ids_from_selector_cache = {}
            for cix_area_path, res_data in list(self.res_index.items()):
                cix_path = AreaResource(cix_area_path).path
                res_id = res_data[0]
                cache[normpath(normcase(cix_path))] = [res_id]
                name = splitext(basename(cix_path))[0].lower()
                if name not in cache:
                    cache[name] = []
                cache[name].append(res_id)
            log.debug("_res_ids_from_selector_cache: %r", cache)

        res_ids = []
        missing_selections = []
        for selector, selection \
                in list(self._selection_from_selector(selections).items()):
            try:
                res_ids += self._res_ids_from_selector_cache[selector]
            except KeyError as ex:
                missing_selections.append(selection)
        log.debug("_res_ids_from_selections: res_ids=%r", res_ids)
        return tuple(res_ids), missing_selections

    @LazyClassAttribute
    def _std_catalog_dir(cls):
        return join(dirname(dirname(abspath(__file__))), "catalogs")

    _catalog_dirs = None

    @property
    def catalog_dirs(self):
        return self._catalog_dirs

    @catalog_dirs.setter
    def catalog_dirs(self, value):
        assert not isinstance(value, str), \
            "catalog_dirs must be an iterable, not a string"
        catalog_dirs = list(value)
        if self._std_catalog_dir not in catalog_dirs:
            catalog_dirs.append(self._std_catalog_dir)
        self._catalog_dirs = catalog_dirs
        # cause a rescan next time we try to get a catalog lib
        self._have_updated_at_least_once = False

    def get_lib(self, lang, selections=None):
        """Return a CatalogLib for the given lang and selections."""
        assert not isinstance(selections, str),\
            "catalog lib 'selections' must be None or a sequence, not %r: %r"\
            % (type(selections), selections)
        if not self._have_updated_at_least_once:
            self.update(selections)

        if selections is not None:
            selection_res_ids, missing_selections \
                = self._res_ids_from_selections(selections)
            if missing_selections:
                self.update(missing_selections)
                selection_res_ids, missing_selections \
                    = self._res_ids_from_selections(selections)
            if missing_selections:
                log.warn("the following catalog selections didn't match "
                         "any loaded API catalog: '%s'",
                         "', '".join(missing_selections))
        else:
            selection_res_ids = None
        key = (lang, selection_res_ids)
        if key not in self._lib_cache:
            self._lib_cache[key] = CatalogLib(self, lang,
                                              selections, selection_res_ids)
        return self._lib_cache[key]

    @property
    def res_index(self):
        """Load and return the resource index (res_index)."""
        if self._res_index is None:
            idxpath = join(self.base_dir, "res_index")
            self._res_index = self.db.load_pickle(idxpath, {})
        return self._res_index

    @property
    def blob_index(self):
        """Load and return the blob index (blob_index)."""
        if self._blob_index is None:
            idxpath = join(self.base_dir, "blob_index")
            self._blob_index = self.db.load_pickle(idxpath, {})
        return self._blob_index

    @property
    def toplevelname_index(self):
        """Load and return the top-level name index (toplevelname_index)."""
        if self._toplevelname_index is None:
            idxpath = join(self.base_dir, "toplevelname_index")
            self._toplevelname_index = self.db.load_pickle(idxpath, {})
        return self._toplevelname_index

    @property
    def toplevelprefix_index(self):
        """Load and return the top-level prefix index (toplevelprefix_index)."""
        if self._toplevelprefix_index is None:
            idxpath = join(self.base_dir, "toplevelprefix_index")
            self._toplevelprefix_index = self.db.load_pickle(idxpath, {})
        return self._toplevelprefix_index

    def save(self):
        self._lock.acquire()
        try:
            for dbsubpath, lpaths in self._dbsubpaths_and_lpaths_to_save:
                self.db.save_pickle(join(self.base_dir, dbsubpath), lpaths)
            self._dbsubpaths_and_lpaths_to_save = []
        finally:
            self._lock.release()

    def cull_mem(self):
        """Drop blobs from cache that have not been accessed in over 5
        minutes.

        To attempt to keep memory consumption under control we want to
        ensure we don't keep everything cached from the db in memory
        until process completion.
        """
        # TOTEST: Does Python/Komodo actually release this memory or
        #        are we kidding ourselves?
        self._lock.acquire()
        try:
            N = 10
            if len(self._blob_and_atime_from_blobname_from_lang_cache) < N:
                # Too few blobs in memory to bother culling.
                return

            log.info("catalog: culling memory")
            now = time.time()
            for lang, blob_and_atime_from_blobname \
                    in list(self._blob_and_atime_from_blobname_from_lang_cache.items()):
                for blobname, (blob, atime) in list(blob_and_atime_from_blobname.items()):
                    if now - atime > 300.0:  # >5 minutes since last access
                        del blob_and_atime_from_blobname[blobname]
        finally:
            self._lock.release()

    def reportMemory(self):
        """
        Report on memory usage from this CatalogsZone.
        @returns {dict} memory usage; keys are the paths, values are a dict of
            "amount" -> number
            "units" -> "bytes" | "count"
            "desc" -> str description
        """
        log.debug("CatalogsZone: reporting memory")
        import memutils

        total_mem_usage = 0
        result = {}
        for lang, blob_and_atime_from_blobname in list(self._blob_and_atime_from_blobname_from_lang_cache.items()):
            for blobname, [blob, atime] in list(blob_and_atime_from_blobname.items()):
                result["explicit/python/codeintel/%s/catalog/%s" % (lang, blobname)] = {
                    "amount": memutils.memusage(blob),
                    "units": "bytes",
                    "desc": "The number of bytes of %s codeintel %s catalog blobs." % (lang, blobname),
                }
        return result

    def avail_catalogs(self, selections=None):
        """Generate a list of available catalogs.

            "selections" (optional) is a list of string of the same form
                as to `.get_lib()'. It is used to determine the boolean
                value of <selected> in the yielded tuples.

        Generated dicts as follows:
            {"name": <catalog-name>,    # 'name' attr of <codeintel> tag
                                        #   or file basename
             "lang": <lang>,            # 'lang' attribute of first <file> tag
             "description": <desc>,     # 'description' attr of <codeintel>
             "cix_path": <cix-path>,
             "selected": <selected>,
             "selection": <selection>,
            }
        where <selected> is boolean indicating if this catalog is
        selected according to "selections" and <selection> is the string
        in "selections" that resulted in this.
        """
        selection_from_selector = self._selection_from_selector(selections)
        for cix_path in (cix_path for d in self.catalog_dirs if exists(d)
                         for cix_path in glob(join(d, "*.cix"))):
            name = lang = description = None
            try:
                for event, elem in ET.iterparse(cix_path, events=("start",)):
                    if elem.tag == "codeintel":
                        name = elem.get("name")
                        description = elem.get("description")
                    elif elem.tag == "file":
                        lang = elem.get("lang")
                        break
            except ET.XMLParserError as ex:
                log.warn("%s: error reading catalog, skipping it (%s)",
                         cix_path, ex)
                continue
            if lang is None:
                log.warn("%s: no 'lang' attribute on catalog <file> tag, "
                         "skipping it", cix_path)
                continue
            if name is None:
                name = splitext(basename(cix_path))[0]
            norm_name = name.lower()
            norm_cix_path = normpath(normcase(cix_path))
            if selection_from_selector is None:
                selected = True
                selection = None
            else:
                selection = (selection_from_selector.get(norm_name)
                             or selection_from_selector.get(norm_cix_path))
                selected = selection is not None
            yield {"name": name,
                   "lang": lang,
                   "description": description,
                   "cix_path": cix_path,
                   "selected": selected,
                   "selection": selection}

    def update(self, selections=None, progress_cb=None):
        """Update the catalog as necessary.

            "selections" (optional) is a list of string of the same form
                as to `.get_lib()' -- used here to filter the catalogs
                that we consider for updating.
            "progress_cb" (optional) is a callable that is called as
                follows to show the progress of the update:
                    progress_cb(<desc>, <value>)
                where <desc> is a short string describing the current step
                and <value> is an integer between 0 and 100 indicating the
                level of completeness.
        """
        self._lock.acquire()
        try:
            self._have_updated_at_least_once = True

            # Figure out what updates need to be done...
            if progress_cb:
                try:
                    progress_cb("Determining necessary catalog updates...", 5)
                except:
                    log.exception("error in progress_cb (ignoring)")
            res_name_from_res_path = dict(  # this is our checklist
                (p, v[2]) for p, v in list(self.res_index.items()))
            todos = []
            log.info("updating %s: %d catalog dir(s)", self,
                     len(self.catalog_dirs))
            for catalog_info in self.avail_catalogs(selections):
                cix_path = catalog_info["cix_path"]
                res = AreaResource(cix_path)
                # check that the update-time is the mtime (i.e. up-to-date)
                try:
                    res_id, last_updated, name, res_data \
                        = self.res_index[res.area_path]
                except KeyError:
                    # add this new CIX file
                    todos.append(("add", res, catalog_info["name"]))
                else:
                    mtime = os.stat(cix_path).st_mtime
                    if last_updated != mtime:  # epsilon? '>=' instead of '!='?
                        # update with newer version
                        todos.append(("update", res, catalog_info["name"]))
                    # else:
                    #    log.debug("not updating '%s' catalog: mtime is unchanged",
                    #              catalog_info["name"])
                    del res_name_from_res_path[res.area_path]  # tick it off

            for res_area_path, res_name in list(res_name_from_res_path.items()):
                # remove this obsolete CIX file
                try:
                    todos.append(("remove", AreaResource(
                        res_area_path), res_name))
                except ValueError as ex:
                    # Skip resources in unknown areas. This is primarily to
                    # allow debugging/testing (when the set of registered
                    # path_areas may not include the set when running in
                    # Komodo.)
                    pass

            # Filter todos on selections, if any.
            if selections is not None:
                selection_from_selector = self._selection_from_selector(
                    selections)
                before = todos[:]
                todos = [todo for todo in todos
                         if todo[2].lower() in selection_from_selector
                         or normpath(normcase(todo[1].path)) in selection_from_selector
                         ]

            # ... and then do them.
            if not todos:
                return
            for i, (action, res, name) in enumerate(todos):
                log.debug("%s `%s' catalog (%s)", action, name, res)
                try:
                    if action == "add":
                        desc = "Adding '%s' API catalog" % basename(
                            res.subpath)
                        if progress_cb:
                            try:
                                progress_cb(desc, (5 + 95/len(todos)*i))
                            except:
                                log.exception(
                                    "error in progress_cb (ignoring)")
                        else:
                            self.db.report_event(desc)
                        self._add_res(res)
                    elif action == "remove":
                        desc = "Removing '%s' API catalog" % basename(
                            res.subpath)
                        if progress_cb:
                            try:
                                progress_cb(desc, (5 + 95/len(todos)*i))
                            except:
                                log.exception(
                                    "error in progress_cb (ignoring)")
                        else:
                            self.db.report_event(desc)
                        self._remove_res(res)
                    elif action == "update":
                        desc = "Updating '%s' API catalog" % basename(
                            res.subpath)
                        if progress_cb:
                            try:
                                progress_cb(desc, (5 + 95/len(todos)*i))
                            except:
                                log.exception(
                                    "error in progress_cb (ignoring)")
                        else:
                            self.db.report_event(desc)
                        # XXX Bad for filesystem. Change this to do it
                        #    more intelligently if possible.
                        self._remove_res(res)
                        self._add_res(res)
                except DatabaseError as ex:
                    log.warn("%s (skipping)" % ex)

            if progress_cb:
                try:
                    progress_cb("Saving catalog indices...", 95)
                except:
                    log.exception("error in progress_cb (ignoring)")
            self._res_ids_from_selector_cache = None  # invalidate this cache
            if self._res_index is not None:
                self.db.save_pickle(
                    join(self.base_dir, "res_index"),
                    self._res_index)
            if self._blob_index is not None:
                self.db.save_pickle(
                    join(self.base_dir, "blob_index"),
                    self._blob_index)
            if self._toplevelname_index is not None:
                self.db.save_pickle(
                    join(self.base_dir, "toplevelname_index"),
                    self._toplevelname_index)
            if self._toplevelprefix_index is not None:
                self.db.save_pickle(
                    join(self.base_dir, "toplevelprefix_index"),
                    self._toplevelprefix_index)
        finally:
            self._lock.release()

    _existing_res_ids_cache = None
    _new_res_id_counter = 0

    def _new_res_id(self):
        if self._existing_res_ids_cache is None:
            self._existing_res_ids_cache \
                = dict((d[0], True) for d in list(self.res_index.values()))
        while True:
            if self._new_res_id_counter not in self._existing_res_ids_cache:
                new_res_id = self._new_res_id_counter
                self._new_res_id_counter += 1
                self._existing_res_ids_cache[new_res_id] = True
                return new_res_id
            self._new_res_id_counter += 1

    def _remove_res(self, res):
        LEN_PREFIX = self.db.LEN_PREFIX
        res_id, last_updated, name, res_data = self.res_index[res.area_path]
        # res_data: {lang -> blobname -> ilk -> toplevelnames}
        for lang, tfifb in list(res_data.items()):
            dbfile_and_res_id_from_blobname = self.blob_index[lang]
            for blobname, toplevelnames_from_ilk in list(tfifb.items()):
                # Update 'blob_index' for $lang.
                dbfile, res_id = dbfile_and_res_id_from_blobname[blobname]
                del dbfile_and_res_id_from_blobname[blobname]

                # Remove ".blob" file (and associated caches).
                pattern = join(self.base_dir, safe_lang_from_lang(lang),
                               dbfile+".*")
                try:
                    for path in glob(pattern):
                        log.debug("fs-write: remove catalog %s blob file '%s'",
                                  lang, basename(path))
                        os.remove(path)
                except EnvironmentError as ex:
                    # XXX If get lots of these, then try harder. Perhaps
                    #    creating a zombies area, or creating a list of
                    #    them: self.db.add_zombie(dbpath).
                    # XXX THis isn't a correct analysis: the dbfile may just
                    #    not have been there.
                    log.warn("could not remove dbfile '%s' (%s '%s'): "
                             "leaving zombie", dbpath, lang, blobname)

                # Update 'toplevel*_index' for $lang.
                # toplevelname_index:   {lang -> ilk -> toplevelname -> res_id -> blobnames}
                # toplevelprefix_index: {lang -> ilk -> prefix -> res_id ->
                # toplevelnames}
                for ilk, toplevelnames in toplevelnames_from_ilk.items():
                    try:
                        bfrft = self.toplevelname_index[lang][ilk]
                        for toplevelname in toplevelnames:
                            del bfrft[toplevelname][res_id]
                            if not bfrft[toplevelname]:
                                del bfrft[toplevelname]
                    except KeyError as ex:
                        self.db.corruption("CatalogsZone._remove_res",
                                           "error removing top-level names of ilk '%s' for "
                                           "'%s' resource from toplevelname_index: %s"
                                           % (ilk, basename(res.path), ex),
                                           "ignore")

                    try:
                        tfrfp = self.toplevelprefix_index[lang][ilk]
                        for toplevelname in toplevelnames:
                            prefix = toplevelname[:LEN_PREFIX]
                            del tfrfp[prefix][res_id]
                            if not tfrfp[prefix]:
                                del tfrfp[prefix]
                    except KeyError as ex:
                        self.db.corruption("CatalogsZone._remove_res",
                                           "error removing top-level name of ilk '%s' for "
                                           "'%s' resource from toplevelprefix_index: %s"
                                           % (ilk, basename(res.path), ex),
                                           "ignore")

        del self.res_index[res.area_path]

    def _add_res(self, res):
        cix_path = res.path
        try:
            tree = tree_from_cix_path(cix_path)
        except ET.XMLParserError as ex:
            log.warn("could not load `%s' into catalog (skipping): %s",
                     cix_path, ex)
            return

        LEN_PREFIX = self.db.LEN_PREFIX
        res_id = self._new_res_id()
        res_data = {}   # {lang -> blobname -> ilk -> toplevelnames}
        name = tree.get("name") or splitext(basename(cix_path))[0]
        for blob in tree.findall("file/scope"):
            lang, blobname = blob.get("lang"), blob.get("name")
            if not lang:
                raise DatabaseError("add `%s': no 'lang' attr on %r"
                                    % (res, blob))

            # Create 'res_data'.
            tfifb = res_data.setdefault(lang, {})
            toplevelnames_from_ilk = tfifb.setdefault(blobname, {})
            if lang in self.db.import_everything_langs:
                for toplevelname, elem in blob.names.items():
                    ilk = elem.get("ilk") or elem.tag
                    if ilk not in toplevelnames_from_ilk:
                        toplevelnames_from_ilk[ilk] = set([toplevelname])
                    else:
                        toplevelnames_from_ilk[ilk].add(toplevelname)

            # Update 'toplevel*_index'.
            # toplevelname_index:   {lang -> ilk -> toplevelname -> res_id -> blobnames}
            # toplevelprefix_index: {lang -> ilk -> prefix -> res_id ->
            # toplevelnames}
            bfrftfi = self.toplevelname_index.setdefault(lang, {})
            tfrfpfi = self.toplevelprefix_index.setdefault(lang, {})
            for ilk, toplevelnames in toplevelnames_from_ilk.items():
                bfrft = bfrftfi.setdefault(ilk, {})
                tfrfp = tfrfpfi.setdefault(ilk, {})
                for toplevelname in toplevelnames:
                    bfr = bfrft.setdefault(toplevelname, {})
                    if res_id not in bfr:
                        bfr[res_id] = set([blobname])
                    else:
                        bfr[res_id].add(blobname)
                    prefix = toplevelname[:LEN_PREFIX]
                    tfr = tfrfp.setdefault(prefix, {})
                    if res_id not in tfr:
                        tfr[res_id] = set([toplevelname])
                    else:
                        tfr[res_id].add(toplevelname)

            # Update 'blob_index'.
            dbfile_and_res_id_from_blobname \
                = self.blob_index.setdefault(lang, {})
            assert blobname not in dbfile_and_res_id_from_blobname, \
                ("codeintel: %s %r blob in `%s' collides "
                    "with existing %s %r blob (from res_id %r) in catalog: "
                    "(XXX haven't decided how to deal with that yet)"
                    % (lang, blobname, cix_path, lang, blobname,
                       dbfile_and_res_id_from_blobname[blobname][1]))
            dbfile = self.db.bhash_from_blob_info(cix_path, lang, blobname)
            dbfile_and_res_id_from_blobname[blobname] = (dbfile, res_id)

            # Write out '.blob' file.
            dbdir = join(self.base_dir, safe_lang_from_lang(lang))
            if not exists(dbdir):
                log.debug("fs-write: mkdir '%s'", dbdir)
                os.makedirs(dbdir)
            log.debug("fs-write: catalog %s blob '%s'", lang, dbfile)
            ET.ElementTree(blob).write(join(dbdir, dbfile+".blob"))

        # Update 'res_index'.
        last_updated = os.stat(cix_path).st_mtime
        self.res_index[res.area_path] \
            = (res_id, last_updated, name, res_data)

    def res_id_from_lang_and_blobname(self, lang, blobname):
        try:
            dbfile, res_id = self.blob_index[lang][blobname]
        except KeyError:
            return None
        else:
            return res_id

    def get_blob(self, lang, blobname, look_in_cache_only=False):
        try:
            dbfile, res_id = self.blob_index[lang][blobname]
        except KeyError:
            return None

        # If index path is in the cache: return it, update its atime.
        now = time.time()
        blob_and_atime_from_blobname \
            = self._blob_and_atime_from_blobname_from_lang_cache.setdefault(lang, {})
        if blobname in blob_and_atime_from_blobname:
            log.debug("cache-read: load %s blob `%s'", lang, blobname)
            blob, atime = blob_and_atime_from_blobname[blobname]
            blob_and_atime_from_blobname[blobname] = (blob, now)
            return blob

        # Need to load and cache it.
        if look_in_cache_only:
            return None
        dbsubpath = join(self.base_dir, safe_lang_from_lang(lang), dbfile)
        blob = self.db.load_blob(dbsubpath)
        blob_and_atime_from_blobname[blobname] = (blob, now)
        return blob

    def lpaths_from_lang_and_blobname(self, lang, blobname):
        """Get lpaths for the named blob.

        We get it from the blob's "lpaths" cache key (calculating that
        if necessary).
        """
        blob = self.get_blob(lang, blobname, look_in_cache_only=True)
        if blob is not None:
            if "lpaths" in blob.cache:
                return blob.cache["lpaths"]
        else:
            blob = self.get_blob(lang, blobname)
            if blob is None:
                raise NotFoundInDatabase("%s '%s' blob not found in catalogs"
                                         % (lang, blobname))
            if "lpaths" in blob.cache:
                return blob.cache["lpaths"]

        # Need to calculate lpaths from 'blob'.
        log.debug("calc symbol info for %s '%s' catalog blob", lang, blobname)
        langintel = self.mgr.langintel_from_lang(lang)
        lpaths = langintel.lpaths_from_blob(blob)

        # Update cache and queue this up to be saved to disk (by .save()).
        blob.cache["lpaths"] = lpaths
        dbfile, res_id = self.blob_index[lang][blobname]
        self._lock.acquire()
        try:
            self._dbsubpaths_and_lpaths_to_save.append(
                (join(safe_lang_from_lang(lang), dbfile+".lpaths"), lpaths)
            )
        finally:
            self._lock.release()

        return lpaths


class CatalogLib(object):
    """A light lang-specific and selection-filtered view on the whole
    CatalogsZone.
    """
    name = "cataloglib"

    def __init__(self, catalogs_zone, lang,
                 selections=None, selection_res_ids=None):
        self.catalogs_zone = catalogs_zone
        self.lang = lang
        self.selections = selections
        if selection_res_ids is None:
            self.selection_res_id_set = None
        else:
            self.selection_res_id_set = set(selection_res_ids)
        self._import_handler = None
        self._blob_imports_from_prefix_cache = {}

    _repr_cache = None

    def __repr__(self):
        if self._repr_cache is None:
            # Include the base names of the selected resources in the name.
            if self.selection_res_id_set is None:
                selection_names = ['(all)']
            else:
                selection_names = []
                for s in self.selections:
                    if isabs(s):
                        selection_names.append(splitext(basename(s))[0])
                    else:
                        selection_names.append(s)
            self._repr_cache = "<%s cataloglib: %s>"\
                               % (self.lang, ', '.join(selection_names))
        return self._repr_cache

    @property
    def import_handler(self):
        if self._import_handler is None:
            self._import_handler \
                = self.catalogs_zone.mgr.citadel.import_handler_from_lang(self.lang)
        return self._import_handler

    def has_blob(self, blobname):
        res_id = self.catalogs_zone.res_id_from_lang_and_blobname(self.lang,
                                                                  blobname)
        if res_id is None:
            return False
        if self.selection_res_id_set is None:
            return True
        return res_id in self.selection_res_id_set

    def get_blob(self, blobname):
        if not self.has_blob(blobname):  # knows how to filter on selections
            return None
        return self.catalogs_zone.get_blob(self.lang, blobname)

    def get_blob_imports(self, prefix):
        """Return the set of imports under the given prefix.

            "prefix" is a tuple of import name parts. E.g. ("xml", "sax")
                for "import xml.sax." in Python. Or ("XML", "Parser") for
                "use XML::Parser::" in Perl.

        See description in database.py docstring for details.
        """
        # This code works fine if prefix is the empty tuple.
        if prefix not in self._blob_imports_from_prefix_cache:
            try:
                dbfile_and_res_id_from_blobname \
                    = self.catalogs_zone.blob_index[self.lang]
            except KeyError:
                return set()

            if self.selection_res_id_set is None:
                matches = filter_blobnames_for_prefix(
                    dbfile_and_res_id_from_blobname,
                    prefix,
                    self.import_handler.sep)
            else:
                matches = filter_blobnames_for_prefix(
                    (bn
                     for bn, (f, res_id) in list(dbfile_and_res_id_from_blobname.items())
                     if res_id in self.selection_res_id_set),
                    prefix,
                    self.import_handler.sep)
            self._blob_imports_from_prefix_cache[prefix] = matches
        return self._blob_imports_from_prefix_cache[prefix]

    def _blobnames_from_toplevelname(self, toplevelname, ilk=None):
        """Yield all blobnames in the currently selected catalogs
        with the given toplevelname.

        If "ilk" is given then only symbols of that ilk will be considered.
        """
        # toplevelname_index: {lang -> ilk -> toplevelname -> res_id ->
        # blobnames}
        if self.lang in self.catalogs_zone.toplevelname_index:
            for i, potential_bfrft \
                    in self.catalogs_zone.toplevelname_index[self.lang].items():
                if ilk is not None and i != ilk:
                    continue
                if toplevelname not in potential_bfrft:
                    continue
                potential_bfr = potential_bfrft[toplevelname]
                if self.selection_res_id_set is None:
                    for blobnames in potential_bfr.values():
                        for blobname in blobnames:
                            yield blobname
                else:
                    for res_id, blobnames in potential_bfr.items():
                        if res_id not in self.selection_res_id_set:
                            continue
                        for blobname in blobnames:
                            yield blobname

    def hits_from_lpath(self, lpath, ctlr=None, curr_buf=None):
        assert isinstance(lpath, tuple)  # common mistake to pass in a string

        hits = []
        for blobname in self._blobnames_from_toplevelname(lpath[0]):
            lpaths = self.catalogs_zone.lpaths_from_lang_and_blobname(
                self.lang, blobname)
            if lpath not in lpaths:
                continue
            blob = self.catalogs_zone.get_blob(self.lang, blobname)
            # TODO: Convert lpath's in tree-evalrs to tuples instead of lists.
            elem = _elem_from_scoperef((blob, list(lpath)))
            hits.append((elem, (blob, list(lpath[:-1]))))

        return hits

    def toplevel_cplns(self, prefix=None, ilk=None, ctlr=None):
        """Return completion info for all top-level names matching the
        given prefix and ilk in all selected blobs in this lib.

            "prefix" is a 3-character prefix with which to filter top-level
                names. If None (or not specified), results are not filtered
                based on the prefix.
            "ilk" is a symbol type (e.g. "class", "variable", "function")
                with which to filter results. If None (or not specified),
                results of any ilk are returned.
            "ctlr" (optional) is an EvalController instance. If
                specified it should be used in the normal way (logging,
                checking .is_aborted()).

        Returns a list of 2-tuples: (<ilk>, <name>).

        Note: the list is not sorted, because often some special sorting
        is required for the different completion evaluators that might use
        this API.
        """
        cplns = []
        if prefix is None:
            # Use 'toplevelname_index':
            #   {lang -> ilk -> toplevelname -> res_id -> blobnames}
            toplevelname_index = self.catalogs_zone.toplevelname_index
            if self.lang in toplevelname_index:
                if ilk is not None:
                    try:
                        bfrft = toplevelname_index[self.lang][ilk]
                    except KeyError:
                        pass
                    else:
                        if self.selection_res_id_set is None:
                            cplns += [(ilk, t) for t in bfrft]
                        else:
                            cplns += [(ilk, t) for t, bfr in bfrft.items()
                                      if self.selection_res_id_set.intersection(bfr)]
                elif self.selection_res_id_set is None:
                    for i, bfrft in toplevelname_index[self.lang].items():
                        cplns += [(i, t) for t in bfrft]
                else:  # ilk=None, have a selection set
                    for i, bfrft in toplevelname_index[self.lang].items():
                        cplns += [(i, t) for t, bfr in bfrft.items()
                                  if self.selection_res_id_set.intersection(bfr)]
        else:
            # Use 'toplevelprefix_index':
            #   {lang -> ilk -> prefix -> res_id -> toplevelnames}
            toplevelprefix_index = self.catalogs_zone.toplevelprefix_index
            if self.lang in toplevelprefix_index:
                if ilk is not None:
                    try:
                        tfr = toplevelprefix_index[self.lang][ilk][prefix]
                    except KeyError:
                        pass
                    else:
                        if self.selection_res_id_set is None:
                            cplns += [(ilk, t)
                                      for toplevelnames in tfr.values()
                                      for t in toplevelnames]
                        else:
                            cplns += [(ilk, t)
                                      for r in self.selection_res_id_set.intersection(tfr)
                                      for t in tfr[r]]
                elif self.selection_res_id_set is None:
                    for i, tfrfp in toplevelprefix_index[self.lang].items():
                        if prefix not in tfrfp:
                            continue
                        cplns += [(i, t)
                                  for toplevelnames in tfrfp[prefix].values()
                                  for t in toplevelnames]
                else:  # ilk=None, have a selection set
                    for i, tfrfp in toplevelprefix_index[self.lang].items():
                        if prefix not in tfrfp:
                            continue
                        tfr = tfrfp[prefix]
                        cplns += [(i, t)
                                  for r in self.selection_res_id_set.intersection(tfr)
                                  for t in tfr[r]]
        return cplns


#---- internal support routines
def _elem_from_scoperef(scoperef):
    """A scoperef is (<blob>, <lpath>). Return the actual elem in
    the <blob> ciElementTree being referred to.  Returns None if not found.
    """
    elem = scoperef[0]
    for lname in scoperef[1]:
        try:
            elem = elem.names[lname]
        except KeyError:
            return None
    return elem
