scripts / FreeIPA

#! /usr/bin/env python3
#
# usersfreeipa.py - python script to migrate/synchronize LDAP users with FreeIPA
#
# Version 1.0, latest version, documentation and bugtracker available at:
#		https://gitlab.lindenaar.net/scripts/freeipa
#
# Copyright (c) 2018 Frederik Lindenaar
#
# This script is free software: you can redistribute and/or modify it under the
# terms of version 3 of the GNU General Public License as published by the Free
# Software Foundation, or (at your option) any later version of the license.
#
# This script is distributed in the hope that it will be useful but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, visit <http://www.gnu.org/licenses/> to download it.

"""
migrate/synchronize LDAP users with FreeIPA.

This script uses LDAP to obtain users from a MacOS Server (or other LDAP)
server and synchronizes the results with the users registered in FreeIPA.
Since it synchronizes data it is safe to run multiple times.

for available command-line options, run 'users2freeipa.py -h'
"""

import os, logging
from argparse import ArgumentParser, FileType, \
              _StoreAction as StoreAction, _StoreConstAction as StoreConstAction
from base64 import b64encode
from getpass import getpass

from ipalib import api
from ipalib.errors import PublicError, AuthenticationError, NotFound

from ldap import LDAPError, SCOPE_SUBTREE, MOD_ADD, MOD_DELETE
from ldap.filter import escape_filter_chars
from ldap.ldapobject import LDAPObject
from ldap.resiter import ResultProcessor


VERSION="1.0"
PROG_NAME=os.path.splitext(os.path.basename(__file__))[0]
PROG_VERSION=PROG_NAME + ' ' + VERSION
LOG_FORMAT='%(levelname)s - %(message)s'
LOG_FORMAT_FILE='%(asctime)s - ' + LOG_FORMAT

logger = logging.getLogger(PROG_NAME)


##################[ Action to immediately set the log level ]##################
class SetLogLevel(StoreConstAction):
    """ArgumentParser action to set log level to provided const value"""
    def __call__(self, parser, namespace, values, option_string=None):
        logging.getLogger(PROG_NAME).setLevel(self.const)

####################[ Action to immediately log to a file ]####################
class SetLogFile(StoreAction):
    """ArgumentParser action to log to file (sets up FileHandler accordingly)"""
    def __call__(self, parser, namespace, values, option_string=None):
        super(SetLogFile, self).__call__(parser,namespace,values,option_string)
        formatter = logging.Formatter(LOG_FORMAT_FILE)
        handler = logging.FileHandler(values)
        handler.setFormatter(formatter)
        logger = logging.getLogger(PROG_NAME)
        logger.propagate = False
        logger.addHandler(handler)


#################[ Generic LDAP Connection / ResultProcessor ]#################
class custom_attr(str):   pass
class LDAPConnection(LDAPObject,ResultProcessor):
    """LDAPConnection implements the LDAP-specifics to migrate LDAP to FreeIPA

    This class holds a generic implementation of the functionality needed to
    migrate from a generic () LDAP server to FreeIPA. It is intended to be
    subclassed to add the instance-specific parameters for implementation.

    Parameters and LDAP attributes steering the migration to FreeIPA are in
    the below attributes (to be overrided by implementation subclasses):

        FreeIPA LDAP Schema customizations (used by check_schema)
            SCHEMA_ATTRIBUTES:    custom attribute mapping (oid:definiton)
            SCHEMA_OBJECTCLASSES: custom objectclass mapping (oid:definiton)

        LDAP search parameters:
            USER_SUBTREE:         LDAP users subtree (relative to baseDN)
            USER_OBJECTCLASS:     objectclasses for searching users
            USER_LOGINATTR:       user login / (unique) key attribute
            USER_MAPPING:         mapping for LDAP attributes to FreeIPA
                                  (LDAP attr: (FreeIPA attr, convert_func))
            GROUP_SUBTREE:        LDAP groups subtree (relative to baseDN)
            GROUP_OBJECTCLASS:    objectclasses for searching group(member)s
            GROUP_NAMEATTR:       group name / (unique) key attribute
            GROUP_DESCATTR:       group description attribute
            GROUP_GIDATTR:        group gid attribute
            GROUP_MEMBERATTR:     group member attribute with member logins
    """
    @staticmethod
    def ldaplist2ipa(valuelist): return tuple(map(bytes.decode, valuelist))

    @staticmethod
    def ldapstr2ipa(valuelist, default=()):
        return ' '.join(map(bytes.decode, valuelist if valuelist else default))

    FREEIPA_SCHEMA_DN = 'cn=schema'
    SCHEMA_ATTRIBUTES = { }
    SCHEMA_OBJECTCLASSES = { }

    USER_OBJECTCLASS = ( 'posixAccount', 'shadowAccount' )

    USER_SUBTREE = 'cn=users'
    USER_LOGINATTR = 'uid'
    USER_MAPPING = {    # based on http://www.zytrax.com/books/ldap/ape/
        USER_LOGINATTR:     (USER_LOGINATTR,    ldaplist2ipa.__func__),
        'userPassword':     ('userpassword',    ldapstr2ipa.__func__),
        'uidNumber':        ('uidnumber',       ldapstr2ipa.__func__),
        'gidNumber':        ('gidnumber',       ldapstr2ipa.__func__),
        'gecos':            ('gecos',           ldapstr2ipa.__func__),
        'homeDirectory':    ('homedirectory',   ldapstr2ipa.__func__),
        'loginShell':       ('loginshell',      ldapstr2ipa.__func__),
        'givenName':        ('givenname',       ldapstr2ipa.__func__),
        'sn':               ('sn',              ldapstr2ipa.__func__),
        'cn':               ('cn',              ldapstr2ipa.__func__),
        'displayname':      ('displayname',     ldapstr2ipa.__func__),
        'initials':         ('initials',        ldapstr2ipa.__func__),
        'title':            ('title',           ldapstr2ipa.__func__),
        'mail':             ('mail',            ldaplist2ipa.__func__),
        'street':           ('street',          ldapstr2ipa.__func__),
        'l':                ('l',               ldapstr2ipa.__func__),
        'st':               ('st',              ldapstr2ipa.__func__),
        'postalCode':       ('postalcode',      ldapstr2ipa.__func__),
        'telephonenumber':  ('telephonenumber', ldaplist2ipa.__func__),
        'mobile':           ('mobile',          ldaplist2ipa.__func__),
        'pager':            ('pager',           ldaplist2ipa.__func__),
        'facsimiletelephonenumber':('facsimiletelephonenumber',ldaplist2ipa.__func__),
        'carlicense':       ('carlicense',      ldaplist2ipa.__func__),
        'ou':               ('ou',              ldapstr2ipa.__func__),
        'departmentNumber': ('departmentnumber',ldaplist2ipa.__func__),
        'employeeNumber ':  ('employeenumber',  ldapstr2ipa.__func__),
        'employeeType':     ('employeetype',    ldapstr2ipa.__func__),
        'manager':          ('manager',         ldapstr2ipa.__func__),
        'userCertificate':  ('usercertificate', ldaplist2ipa.__func__),
        'preferredLanguage':('preferredlanguage',ldapstr2ipa.__func__),
    }

    GROUP_OBJECTCLASS = ( 'posixGroup', )
    GROUP_SUBTREE = 'cn=groups'
    GROUP_NAMEATTR = 'cn'
    GROUP_DESCATTR = GROUP_NAMEATTR
    GROUP_GIDATTR = 'gidNumber'
    GROUP_MEMBERATTR = 'memberUid'

    @staticmethod
    def attrfilters(valuelist, attr, cmp='='):
        """convert a value list into an LDAP-escaped attr=value filter list"""
        return set(map(lambda x:attr + cmp + escape_filter_chars(x), valuelist))

    @staticmethod
    def filters2str(*filters, operator, default=''):
        """convert a list of filters to a single filter(string) with operator"""
        result = set('(%s)' % f if f[0]!='(' else f for f in filters if f)
        return '(' + operator + ''.join(result) + ')' if len(result) > 1 else \
                result.pop() if result else default

    @staticmethod
    def notfilter(fstr):
        """Create a negative (NOT) LDAP filter based on a filter (string)"""
        return '' if not fstr else ('(!%s)' if fstr[0]=='(' else '(!(%s))')%fstr

    def objectclassfilter(self, *objclasses, operator='|', cmp='=', default=''):
        """Generates an LDAP filter to retrieve specific objectclass(es)"""
        objclsfilter = self.attrfilters(objclasses, 'objectClass', cmp)
        return self.filters2str(*objclsfilter,operator=operator,default=default)

    def usersfilter(self, *users, operator='|', cmp='=', default=''):
        """Generates an LDAP filter to retrieve users based on login name(s)"""
        userfilter = self.attrfilters(users,self.USER_LOGINATTR,cmp)
        return self.filters2str(*userfilter, operator=operator, default=default)

    def groupsfilter(self, *groups, operator='|', cmp='=', default=''):
        """Generates an LDAP filter to retrieve groups based on group name(s)"""
        groupfilter = self.attrfilters(groups, self.GROUP_NAMEATTR, cmp)
        return self.filters2str(*groupfilter, operator=operator, default=default)

    def groupmemberfilter(self, *members, operator='|', cmp='=', default=''):
        """Generates an LDAP filter to retrieve groups based on their members"""
        memberfilter = self.attrfilters(members, self.GROUP_MEMBERATTR, cmp)
        return self.filters2str(*memberfilter,operator=operator,default=default)

    def __init__(self, server, basedn=None):
        """Initialize connection to server, get server basedn unless provided"""
        super(LDAPConnection, self).__init__(server)
        self._basedn = basedn if basedn else self.get_naming_contexts()[0].decode()
        logger.debug('Connected to %s, base_dn=%s', server, self._basedn)

    def check_schema(self, exists=True, update_schema=False):
        """Check if schema customizations are present in the FreeIPA LDAP
           server (or not, depending on 'exists') and apply the changes if
           'update_schema'. Returns whether the desired state is present.
        """
        if not self.SCHEMA_ATTRIBUTES and not self.SCHEMA_OBJECTCLASSES:
            return True
        logger.debug('Checking LDAP schema customizations presence in FreeIPA')
        operation = MOD_ADD if exists else MOD_DELETE
        config = { 'attributeTypes': self.SCHEMA_ATTRIBUTES,
                    'objectClasses': self.SCHEMA_OBJECTCLASSES }
        ipa_ldap = LDAPObject('ldap:///')
        ipa_schema = ipa_ldap.read_s(self.FREEIPA_SCHEMA_DN, attrlist=config.keys())
        updates = [ (operation, key, value) for key, cfg in config.items()
            for cur in ({v.split(b' ')[1]: v for v in ipa_schema[key]},)
                for value in ([ d for o,d in cfg.items() if o not in cur ] \
                    if exists else [cur[o] for o in cfg if o in cur],) if value]
        if updates and update_schema:
            updates = sorted(updates, key=lambda x: x[1], reverse=not exists)
            logger.info('%sing LDAP schema customizations in FreeIPA',
                                                'Add' if exists else 'Delet')
            logger.debug('LDAP Schema changes: %s', updates)
            ipa_ldap.bind_s('cn=directory manager',
                                        getpass('FreeIPA diradmin password: '))
            ipa_ldap.modify_s(self.FREEIPA_SCHEMA_DN, updates)
        elif updates:
            logger.debug('Not %sing LDAP schema customizations in FreeIPA',
                                                'add' if exists else 'delet')
        else:
            logger.debug('LDAP schema customizations are %s in FreeIPA',
                                        'present' if exists else 'absent')
        ipa_ldap.unbind()
        return(not updates or update_schema)

    def search(self, base, *args, relativebase=True, **kwargs):
        """Search using LDAPObject.search(), adding baseddn to base if needed"""
        if relativebase and not base.endswith(self._basedn):
            base += ',' + self._basedn
        return super(LDAPConnection, self).search(base, *args, **kwargs)

    def get_group_members(self, *group):
        """Returns set with users that are member of the provided group(s)"""
        if not group:
            return set()
        filter=self.filters2str(self.objectclassfilter(*self.GROUP_OBJECTCLASS),
                                self.groupsfilter(*groups), operator='&')
        logger.debug('Searching %s members with LDAP filter: %s',group,filter)
        attrs = (self.GROUP_MEMBERATTR,)
        msg_id = self.search(self.GROUP_SUBTREE, SCOPE_SUBTREE, filter, attrs)
        return set(user.decode() for _,res_data,_,_ in ldap.allresults(msg_id)
                                    for _, entry in res_data
                                        for userlist in entry.values()
                                            for user in userlist)

    def get_user_groups(self, user, groups=()):
        """Returns list of groups that the specified user is member of.
           checks the list of groups, if provided, or all groups
        """
        filter=self.filters2str(self.objectclassfilter(*self.GROUP_OBJECTCLASS),
                                self.groupmemberfilter(user),
                                self.groupsfilter(*groups), operator='&')
        logger.debug('Searching user\'s groups with LDAP filter: %s', filter)
        attrs = (self.GROUP_NAMEATTR,self.GROUP_GIDATTR,self.GROUP_DESCATTR)
        msg_id = self.search(self.GROUP_SUBTREE, SCOPE_SUBTREE, filter, attrs)
        return [ (self.ldapstr2ipa(entry[self.GROUP_NAMEATTR]),
                    self.ldapstr2ipa(entry[self.GROUP_GIDATTR]),
                    self.ldapstr2ipa(entry.get(self.GROUP_DESCATTR,
                                                entry[self.GROUP_NAMEATTR])) )
                            for _,res_data,_,_ in ldap.allresults(msg_id)
                                    for dn, entry in res_data ]

    def get_users(self, *users, excl_users=(), customfilter='', attrs=None):
        """Returns an generator to iterate over LDAP users mapped to FreeIPA.
           This function obtains the list of users (all if not specified) and
           support exclusion list and custom filters for further control
        """
        filter=self.filters2str(self.objectclassfilter(*self.USER_OBJECTCLASS),
                                customfilter, self.usersfilter(*users),
                                self.notfilter(self.usersfilter(*excl_users)),
                            operator='&', default='(%s=*)'%self.USER_LOGINATTR)
        logger.debug('Searching users with LDAP filter: %s', filter)
        msg_id = self.search(self.USER_SUBTREE, SCOPE_SUBTREE, filter, attrs)
        for res_type,res_data,res_msgid,res_controls in self.allresults(msg_id):
            for dn, entry in res_data:
                if logger.getEffectiveLevel() <= logging.DEBUG:
                    notmigrated = ', '.join(key for key in entry if key not in
                                    (*self.USER_MAPPING.keys(), 'objectClass'))
                    if notmigrated:
                        logger.debug('Ignoring attributes: %s', notmigrated)
                yield (dn, { ipa_key: conv(v) for k, v in entry.items()
                            if k in self.USER_MAPPING
                                for ipa_key, conv in (self.USER_MAPPING[k],) })



##################[ MacOS LDAP Connection / ResultProcessor ]##################
class MacOSLDAPConnection(LDAPConnection):
    """LDAPConnection subclass to migrate from Apple's OpenDirectory"""

    # @staticmethod
    # def binary2ipa(valuelist): return b64encode(valuelist[0]).decode()

    SCHEMA_ATTRIBUTES = {
        # Definition (and OID) taken from Apple MacOS Server OpenDirectory
        b"1.3.6.1.4.1.63.1000.1.1.1.1.20":
            b"( 1.3.6.1.4.1.63.1000.1.1.1.1.20 NAME ( 'apple-generateduid' )"
            b" DESC 'generated unique ID' X-ORIGIN 'Apple Server OpenDirectoy'"
            b" EQUALITY caseExactMatch SUBSTR caseExactSubstringsMatch"
            b" SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )",
    }
    SCHEMA_OBJECTCLASSES = {
        # OID: iso.org.dod.internet.private.enterprise.lindenaar.2018.8.2.1.1
        # https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers
        b"1.3.6.1.4.1.18143.2018.8.2.1.1":
            b"( 1.3.6.1.4.1.18143.2018.8.2.1.1"
            b" NAME 'openDirectoryPerson' SUP person STRUCTURAL"
            b" MAY ( c $ apple-generateduid ) X-ORIGIN 'Extending FreeIPA' )"
    }

    USER_OBJECTCLASS = ( *LDAPConnection.USER_OBJECTCLASS, 'apple-user' )
    USER_MAPPING = { **LDAPConnection.USER_MAPPING,
        'objectClass':        (custom_attr('objectclass'), lambda x: 'openDirectoryPerson'),
        'c':                  (custom_attr('c'),         LDAPConnection.ldapstr2ipa),
        'apple-generateduid': (custom_attr('apple-generateduid'),LDAPConnection.ldapstr2ipa),
        'apple-user-homeurl': LDAPConnection.USER_MAPPING['homeDirectory'],
        # TODO: MacOS attribute: jpegPhoto
        # 'jpegPhoto':          (custom_attr('jpegphoto'), binary2ipa.__func__),
        # TODO: migrate MacOS Passwords (not sure this is possible)
    }

    GROUP_OBJECTCLASS = ( *LDAPConnection.GROUP_OBJECTCLASS, 'apple-group' )
    GROUP_DESCATTR = 'apple-group-realname'


###########[ Generic function to get/create/update FreeIPA entries ]###########
def ipa_update_create(api,entity,name,*keys,get_params={},add_params={},**data):
    """ Generic function to create or update an entity and return it.
        This will create or update the entity, with key *key so that it reflects
        the values provided in the mapping **data.
        Parameters:
            api:        FreeIPA API object to use
            entity:     name of the FreeIPA entity type to create/update
            name:       textual name of the entity for logging
            keys:       the FreeIPA key for this entity (1 or more values)
            get_params: additional parameters when retrieving the entity
            add_params: additional parameters/values when creating the entity
            data:       mapping containing the desired values for the entity
        Returns the updated entity
    """
    try:
        ipa_data = getattr(api.Command, '%s_show' % entity)(*keys, **get_params)['result']
        if data:
            updates = { k:v for k, v in data.items() if not (k in ipa_data and v in ipa_data[k]) }
            if updates:
                logger.info('updating existing %s %s in FreeIPA', name, keys[-1])
                return getattr(api.Command, '%s_mod' % entity)(*keys, **updates)['result']
            else:
                logger.debug('no updates for existing %s %s in FreeIPA', name, keys[-1])
        else:
            logger.debug('found existing %s %s in FreeIPA', name, keys[-1])
        return ipa_data
    except NotFound:
        logger.info('creating new %s %s in FreeIPA', name, keys[-1])
        return getattr(api.Command, '%s_add' % entity)(*keys, **data, **add_params)['result']


##########################[ Command-line processing ]##########################
def parse_args():
    """Parse command line and get parameters from environment if not set"""
    parser = ArgumentParser(
        description='Migrate or synchronize LDAP users with FreeIPA',
        epilog='(*) unless specified FreeIPA defaults are used. %s is replaced '
                'by the userID'
    )
    parser.add_argument('-V', '--version',action="version",version=PROG_VERSION)

    pgroup = parser.add_mutually_exclusive_group(required=False)
    pgroup.add_argument('-O', '--apple-opendirectory',   action='store_const',
                        dest='ldapconnection', const=MacOSLDAPConnection,
                        default=LDAPConnection,
                        help='migrate from Apple\'s MacOS Server OpenDirectory')

    parser.add_argument('ldapserver',  help='LDAP Server to request users from')
    parser.add_argument('-b', '--basedn', help='Base DN of source LDAP server, defaults to LDAP server')
    parser.add_argument('-u', '--user', nargs='+',  default=(),
                        help='user(s) to migrate/synchronize, defaults to all')
    parser.add_argument('-g', '--member-of', nargs='+', dest='user_group', default=(),
                        help='migrate/synchronize members of group(s)')
    parser.add_argument('-x', '--exclude-user', nargs='+',  dest='x_user', default=(),
                        help='user(s) to exclude')
    parser.add_argument('-X', '--exclude-group', nargs='+',  dest='x_group', default=(),
                        help='exclude users in specified group(s)')
    parser.add_argument('-f', '--filter', help='additional LDAP filter to apply')
    parser.add_argument('-G', '--migrate-groups', nargs='*', dest='group',
                        help='migrate groups+membership for specified or all groups')
    parser.add_argument('-H', '--migrate-homedir', nargs='?',  dest='homedir', default="",
                        help='Migrate (if empty) or set homedir (*)')
    parser.add_argument('-S', '--migrate-loginshell', nargs='?',  dest='loginshell', default="",
                        help='Migrate (if empty) or set login shell (*)')
    parser.add_argument('-P', '--migrate-password', action='store_true',
                        help='migrate passwords from LDAP (if present)')
    parser.add_argument('-p', '--enforce-passwords', dest='password_logfile',
                        help='enforce users have passwords, file generated passwords')
    parser.add_argument('-s', '--stage', action='store_true',
                        help='create new users as staged users')
    parser.add_argument('-c', '--compatibility-view', dest='idview',
                        help='Maintain ID Compatibility view to preserve UID/GID/homedir/shell')
    parser.add_argument('-U', '--update-schema', action='store_true',
                        help='Configure necessary LDAP schema customizations')
    parser.add_argument('-C', '--cleanup-schema', action='store_true',
                        help='Remove applied LDAP schema customizations and exit')

    pgroup = parser.add_mutually_exclusive_group(required=False)
    pgroup.add_argument('-q', '--quiet',   action=SetLogLevel, const=logging.CRITICAL,
                        default=logging.CRITICAL, help='quiet (only fatal errors)')
    pgroup.add_argument('-v', '--verbose', action=SetLogLevel, const=logging.INFO,
                        help='more verbose output')
    pgroup.add_argument('-d', '--debug',   action=SetLogLevel, const=logging.DEBUG,
                        help='debug output')
    parser.add_argument('-l', '--logfile', action=SetLogFile,
                        help='send logging output to logfile')

    return parser.parse_args()


############################[ LDAPUser processing ]############################
def process_user(api, userid, user_data, stage=False, enforce_password=False):
    """Process a single LDAP user and synchronize with FreeIPA using its API"""
    logger.info('processing LDAP user %s', userid)

    try:
        ipa_data = api.Command.user_show(userid, all=True)['result']
        logger.debug('found active user %s', userid)
        stage = False
    except NotFound:
        try:
            ipa_data = api.Command.stageuser_show(userid, all=True)['result']
            if stage:
                logger.debug('found existing stage entry for user %s', userid)
            else:
                logger.info('activating stage user %s', userid)
                api.Command.stageuser_activate(userid)
        except NotFound:
            ipa_data = { }

    what = 'stageuser' if stage else 'user'
    updates = user_data if not ipa_data else { key: value
                for key, value in user_data.items() if not (key in ipa_data and
                                value in ipa_data[key] if isinstance(value, str)
                                else set(value).issubset(ipa_data[key])) }

    if ipa_data.get('has_password') and 'userpassword' in updates:
        logger.debug('Not replacing existing password for %s %s', what, userid)
        del updates['userpassword']
    elif enforce_password and not (stage or ipa_data.get('has_password')):
        logger.info('Enforcing password for user %s in FreeIPA', userid)
        updates['random'] = True

    if updates or not ipa_data:
        action = 'Upd' if ipa_data else 'Cre'
        logger.info('%sating %s %s in FreeIPA', action, what, userid)
        logger.debug('%sating %s %s with %s', action, what, userid, updates)
        cust=[k+'='+v for k,v in updates.items() if isinstance(k, custom_attr)]
        if cust:
            updates = { 'addattr': cust if ipa_data else None, **{ k:v
                for k,v in updates.items() if not isinstance(k, custom_attr) }}
        command = getattr(api.Command, what + ('_mod' if ipa_data else '_add'))
        ipa_data = command(userid, **updates)['result']
        if cust and action == 'Cre':
            getattr(api.Command, what + '_mod')(userid, addattr=cust)
    else:
        logger.debug('no updates for %s %s in FreeIPA', what, userid)

    return ipa_data.get('memberof_group', ()), stage, \
                                                ipa_data.get('randompassword')


#########################[ Get FreeIPA group members ]#########################
def get_group(api, group, gid, description, idview, group_cache={}):
    try:
        return group_cache[group]
    except KeyError:
        ipa_group = ipa_update_create(api, 'group', 'group', group,
                                    add_params={'description': description})
        result = group_cache[group] = ipa_group.get('member_user', [])

        if idview: # Update or create idoverride for group?
            ipa_update_create(api, 'idoverridegroup', 'GID override', idview,
                              group, gidnumber=gid, cn=group, add_params={
                  'description': "Added by %s while syncronizing %s with %s" \
                                      % (PROG_NAME, group, args.ldapserver) })
        return result


######################[ Update FreeIPA group membership ]######################
def group_commit(command, desc, group, updates):
    logger.info('Updating FreeIPA group %s, %s: %s', group, desc, updates)
    getattr(api.Command, command)(group, user=list(updates))

def group_queue(command, desc, groups, users, commit, maxqueue=0, queue={}):
    if groups and users:
        for group in (groups,) if isinstance(groups, str) else groups:
            try:
                queued = queue[group]
                queued.update((users,) if isinstance(users, str) else users)
                if not commit and len(queued) > maxqueue:
                    group_commit(command, desc, group, queued)
                    del queue[group]
            except KeyError:
                queue[group] = set((users,) if isinstance(users,str) else users)
    if commit and queue:
        for group, updates in queue.items():
            group_commit(command, desc, group, updates)
        queue.clear()

def group_add(api, groups=None, users=None, commit=False, maxq=100, queue={}):
    group_queue('group_add_member', 'adding', groups, users, commit, maxq,queue)

def group_del(api, groups=None, users=None, commit=False, maxq=100, queue={}):
    group_queue('group_remove_member','removing',groups,users,commit,maxq,queue)


###############################################################################
def attr_replace(data, attr, value, param=None):
    """Remove/overwrite data[option] based on value, return inital data[option].
       parameters:
         data - dict to update
         option - attribute in data to update
         value - if ''-->delete, if None-->no change, else overwrite with value
         param - if value contains %s, it will be replaced by this value
    """
    result = data.get(attr)
    if value:
        data[attr] = value % userid if '%s' in value else value
    elif value == '' and result is not None:
        del data[attr]
    return result
def attr_remove(data, attr):  return attr_replace(data, attr, '', None)


if __name__ == '__main__':
    logging.basicConfig(format=LOG_FORMAT)
    args = parse_args()

    try:
        logger.debug('connecting to FreeIPA')
        if api.isdone('finalize') is False:
            api.bootstrap_with_global_options(context='api')
            api.finalize()
        api.Backend.rpcclient.connect()

        logger.info('Connecting to LDAP server %s', args.ldapserver)
        ldap = args.ldapconnection(args.ldapserver, args.basedn)

        if not ldap.check_schema(exists=not args.cleanup_schema,
                    update_schema=args.update_schema or args.cleanup_schema):
            logger.error("Required LDAP schema customizations not in FreeIPA")
            exit(1)
        elif args.cleanup_schema:
            print("LDAP schema customizations are removed from FreeIPA")
            exit(0)

        if args.idview:     # Ensure IDView exists (if specified)
            ipa_update_create(api, 'idview', 'Compatibility View', args.idview,
                add_params={ 'description': 'Created by %s while syncronizing '
                                    'with %s' % (PROG_NAME, args.ldapserver) })

        logger.info('searching for users')
        excl_users = ldap.get_group_members(*args.x_group) | set(args.x_user)
        users = ldap.get_group_members(*args.user_group) | set(args.user)
        for dn, user_data in ldap.get_users(*users, excl_users=excl_users,
                                                    customfilter=args.filter):
            user, *aliases = attr_remove(user_data, 'uid')
            if aliases:
                logger.warn('ignoring additional userid\'s for user %s: %s',
                                                        user,', '.join(aliases))
            if not args.migrate_password:
                attr_remove(user_data, 'userpassword')
            uid=attr_remove(user_data, 'uidnumber')
            gid=attr_remove(user_data, 'gidnumber')
            homedir=attr_replace(user_data,'homedirectory',args.homedir,user)
            loginshell=attr_replace(user_data,'loginshell',args.loginshell,user)

            in_groups, is_stage, password = process_user(api, user, user_data,
                stage=args.stage, enforce_password=bool(args.password_logfile))

            if password:
                with open(args.password_logfile, 'a') as file:
                    print(user, password, user_data.get('givenname',''),
                        user_data.get('sn',''), user_data.get('cn',''),
                        ','.join(user_data.get('mail','')), sep='\t', file=file)

            if is_stage:
                logger.debug('not processing group membership or ID overrides '
                                'for stage user %s', user)
                continue

            if args.idview:     # Update or create the idoverride for the user?
                ipa_update_create(api, 'idoverrideuser', 'UID/GID override',
                            args.idview, user, uidnumber=uid, gidnumber=gid,
                            homedirectory=homedir, loginshell=loginshell,
                            add_params={ 'description':
                                    "Added by %s while syncronizing %s with %s"
                                        % (PROG_NAME, user, args.ldapserver)})

            if args.group is not None:
                remaining_groups = list(in_groups) if not args.group else \
                                    [ g for g in in_groups if g in args.group ]
                for group, gid, desc in ldap.get_user_groups(user, args.group):
                    if user not in get_group(api,group,gid,desc,args.idview):
                        logger.debug('adding user %s to group %s', user, group)
                        group_add(api, group, user)
                    if group in remaining_groups:
                        remaining_groups.remove(group)
                if remaining_groups:
                    logger.debug('removing user %s from group(s) %s', user,
                                                    ', '.join(remaining_groups))
                    group_del(api, remaining_groups, user)

        group_add(api, commit=True)
        group_del(api, commit=True)
    except LDAPError as e:
        logger.critical("Error with LDAP commmunication: %s", e)
    except AuthenticationError:
        logger.critical("Unable to authenticate to FreeIPA, "
                                "make sure you have a valid Kerberos ticket!")
    except PublicError as e:
        logger.critical("error while communicating with FreeIPA: %s", e)
    else:
        exit(0)
    exit(1)