Index: libcfs/include/libcfs/bitmap.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/bitmap.h,v
retrieving revision 1.4
diff -u -p -r1.4 bitmap.h
--- libcfs/include/libcfs/bitmap.h	8 Sep 2009 08:09:26 -0000	1.4
+++ libcfs/include/libcfs/bitmap.h	1 Jan 2010 20:03:30 -0000
@@ -36,15 +36,77 @@
 #ifndef _LIBCFS_BITMAP_H_
 #define _LIBCFS_BITMAP_H_
 
+/** \addtogroup libcfs
+ * @{ */
 
+/**
+ * \defgroup libcfs_bitops Bitmask
+ * Generic API for bitmask operations
+ *
+ * The API operates on \a bitmasks - objects aligned to the size of an
+ * "unsigned long" type, and are least of that size. The endianness
+ * of the bits within each "unsigned long" is the native endianness
+ * of the cpu.
+ *
+ * A bitmask may be regarded as numbered set of bits: every bit has a
+ * number equal to its position in bitmask. This allows to perform
+ * main operations using bit number as specification of bit to operate
+ * on. Main operations include:
+ * - set, clear and test bit
+ * - find first zero or non-zero bit starting from the beginning of
+ * bitmask or from specified position
+ *
+ * There are two flavors of API currently provided:
+ *
+ * 1. Low-level API that deals with bitmasks directly. It includes
+ * primitives set_bit, clear_bit, test_bit, find_first_bit, etc. To
+ * use this API, the caller should allocate and maintain bitmask
+ * objects on her own, passing (unsigned long *) pointer to bitmask
+ * and its size to the primitives.
+ *
+ * 2. Hi-level API. It abstracts bitmask object as \a bitmap_t and
+ * provides separate primitives for its allocation and freeing:
+ * ALLOCATE_BITMAP and FREE_BITMAP. After an object of \a bitmap_t
+ * type is allocated, it may be used as argument for subsequent
+ * calls. The names of primitives of this "hi-level" API start from
+ * cfs_ prefix: cfs_bitmap_set, cfs_bitmap_clear, etc.
+ *
+ * Bitmask primitives may be used for many purposes:
+ * - A bitmask may play the role of storage for various flags
+ * reflecting the state of some subsystem (e.g. "RUNNING", "STOPPED",
+ * etc.)
+ * - Tracking business of limited set of resources. If correspondent
+ * bit is set, the resource with given number is "busy" and vice
+ * versa.
+ * - Dynamic numbering of resources. For example, when a client is
+ * connecting to us at first time, find_first_zero_bit may be used to
+ * associate the client with this first unused bit providing the
+ * number for this client. After the number is acquired, the caller
+ * usually sets this bit to mark it as "busy". So, next client will get
+ * next available number.
+ *
+ * NOTE: Currently, Libcfs doesn't guarantee atomicity of operations
+ * like test_and_set_bit on all platforms. Some platforms (like linux
+ * kernel) implement them in atomic way, while others (like
+ * user-space) don't not.
+ * @{
+ */
+
+/**
+ * Generic bitmap - bitmask along with its size
+ */
 typedef struct {
+        /** size of bitmask (in bits) */
         int             size;
+        /** bitmask */
         unsigned long   data[0];
 } bitmap_t;
 
+/** # bytes to afford bitmap_t with bitmask having at least \a bits+1 bits */
 #define CFS_BITMAP_SIZE(nbits) \
      (((nbits/BITS_PER_LONG)+1)*sizeof(long)+sizeof(bitmap_t))
 
+/** allocate bitmap_t with bitmask having at least \a size+1 bits */
 static inline
 bitmap_t *ALLOCATE_BITMAP(int size)
 {
@@ -79,22 +141,28 @@ int cfs_bitmap_check(bitmap_t *bitmap, i
         return test_bit(nbit, bitmap->data);
 }
 
+/** clear bit \a nbit in bitmap \a bitmap
+ * \return previous value of bit \a nbit */
 static inline
 int cfs_bitmap_test_and_clear(bitmap_t *bitmap, int nbit)
 {
         return test_and_clear_bit(nbit, bitmap->data);
 }
 
-/* return 0 is bitmap has none set bits */
+/** return 0 if bitmap has none set bits */
 static inline
 int cfs_bitmap_check_empty(bitmap_t *bitmap)
 {
         return find_first_bit(bitmap->data, bitmap->size) == bitmap->size;
 }
 
+/** Iterates through all non-zero bits in \a bitmap. On each iteration,
+ * \a pos is the number of current non-zero bit (counting from 0) */
 #define cfs_foreach_bit(bitmap, pos) \
 	for((pos)=find_first_bit((bitmap)->data, bitmap->size);   \
             (pos) < (bitmap)->size;                               \
             (pos) = find_next_bit((bitmap)->data, (bitmap)->size, (pos)))
 
+/** @} libcfs_bitops */
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/curproc.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/curproc.h,v
retrieving revision 1.4
diff -u -p -r1.4 curproc.h
--- libcfs/include/libcfs/curproc.h	9 Sep 2008 17:52:25 -0000	1.4
+++ libcfs/include/libcfs/curproc.h	1 Jan 2010 20:03:30 -0000
@@ -43,12 +43,30 @@
 #ifndef __LIBCFS_CURPROC_H__
 #define __LIBCFS_CURPROC_H__
 
-#ifdef __KERNEL__
-/*
- * Portable API to access common characteristics of "current" UNIX process.
+/** \addtogroup libcfs
+ * @{ */
+
+/** \defgroup libcfs_curproc Curproc
+ * Generic API to access common characteristics of "current" UNIX process.
  *
- * Implemented in portals/include/libcfs/<os>/
+ * Generic API includes primitives for manipulating capabilities (what an
+ * user is capable to do and what she's not), constant CFS_CURPROC_COMM_MAX and
+ * cfs_curproc_uid. All other curproc primitives are platform-specific add-ons.
+ * @{
+ * \defgroup libcfs_kernel_curproc Curproc (kernel-space)
+ * An extension to generic curproc API for kernel users.
+ * For kernel users, Libcfs provides additional curproc primitives for
+ * accessing to process's gid, fsuid, fsgid, groups membership, umask and
+ * world-visible name of the process.
  */
+
+/* XXX Maxim: The curproc API is currently complete mess containing a lot of
+ * entries that nobody use. To be re-designed to have _single_ clean API for all
+ * cases (kernel/user-space, linux/others). */
+
+#ifdef __KERNEL__
+/** \addtogroup libcfs_kernel_curproc
+ * @{ */
 uid_t  cfs_curproc_uid(void);
 gid_t  cfs_curproc_gid(void);
 uid_t  cfs_curproc_fsuid(void);
@@ -70,8 +88,12 @@ char  *cfs_curproc_comm(void);
  *
  * cfs_kernel_cap_t
  */
+/** @} libcfs_kernel_curproc */
 #endif
 
+/** This type is actually a bitmask where each particular bit may represent some
+ * "capability" - an action (or group of actions) that the user is capable to
+ * perform. */
 typedef __u32 cfs_cap_t;
 
 #define CFS_CAP_CHOWN                   0
@@ -101,6 +123,9 @@ cfs_cap_t cfs_curproc_cap_pack(void);
 void cfs_curproc_cap_unpack(cfs_cap_t cap);
 int cfs_capable(cfs_cap_t cap);
 
+/** @} libcfs_curproc */
+/** @} libcfs */
+
 /* __LIBCFS_CURPROC_H__ */
 #endif
 /*
Index: libcfs/include/libcfs/libcfs.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs.h,v
retrieving revision 1.12
diff -u -p -r1.12 libcfs.h
--- libcfs/include/libcfs/libcfs.h	1 Dec 2009 02:23:42 -0000	1.12
+++ libcfs/include/libcfs/libcfs.h	1 Jan 2010 20:03:31 -0000
@@ -37,6 +37,157 @@
 #ifndef __LIBCFS_LIBCFS_H__
 #define __LIBCFS_LIBCFS_H__
 
+/**
+ * \defgroup libcfs Libcfs
+ *
+ * The Lustre library of data types, routines and macros of common use.
+ *
+ * Libcfs is a key component of Lustre file-system abstracting other
+ * components form peculiarities of operation environment like memory
+ * allocation, synchronization and logging facilities.
+ * @{
+ */
+
+/**
+ * \defgroup libcfs_types Generic types
+ *
+ * Generic integer and pointer-based types and primitives for swabbing.
+ *
+ * Libcfs replenishes the lack of generic type definitions (like __u64
+ * and long_ptr_t) on some platforms defining them in
+ * platform-specific libcfs headers. It also defines macros for
+ * endianness conversion (like __swab32, le32_to_cpu and many others)
+ * on platforms where such macros are not present in system headers.
+ */
+
+/**
+ * \defgroup libcfs_procfs Proc file system and sysctl
+ *
+ * Generic API for manipulating /proc entries and sysctl tables
+ *
+ * 1. /proc API
+ *
+ * The API includes:
+ * - types of read and write callbacks that are called from kernel
+ * when user reads from or writes to some file under /proc subtree
+ * (\a cfs_read_proc_t and \a cfs_write_proc_t)
+ * - type describing /proc entry itself (\a cfs_proc_dir_entry_t)
+ * - primitives to register and unregister an entry
+ * (\a cfs_create_proc_entry and \a cfs_remove_proc_entry)
+ * - primitive to free unused entry (\a cfs_free_proc_entry).
+ *
+ * Using this API, the consumer implements read and/or write
+ * callbacks and register entry with \a cfs_create_proc_entry that
+ * returns the pointer to newly created entry of \a cfs_proc_dir_entry_t
+ * type. Then, the consumer assigns its read and write callbacks to
+ * \a read_proc and \a write_proc fields of the entry correspondingly.
+ *
+ * To delete an entry from /proc and free it, the consumer must use
+ * \a cfs_remove_proc_entry. \a cfs_free_proc_entry is dangerous and
+ * should never be used directly.
+ *
+ * 2. sysctl API
+ *
+ * The API includes:
+ * - type describing sysctl table itself (\a cfs_sysctl_table_t)
+ * - type to be used as handle for registered sysctl table
+ * (\a cfs_sysctl_table_header_t)
+ * - primitives to register and unregister sysctl table
+ * (\a cfs_register_sysctl_table and \a cfs_unregister_sysctl_table)
+ * - macro alleviating definitions of sysctl callbacks
+ * (\a LL_PROC_PROTO)
+ *
+ * Using this API, the consumer implements sysctl callback starting
+ * implementation from "int LL_PROC_PROTO(name_of_callback) {". Then,
+ * defines sysctl table of \a cfs_sysctl_table_t type enumerating its
+ * elements one by one. Usually, each element at least specifies the
+ * address of data (on which given sysctl will operate) and
+ * correspondent callback to handle read/write operation (what was
+ * implemented with \a LL_PROC_PROTO macro).
+ *
+ * Having sysctl table defined, the consumer makes it visible for
+ * users by calling \a cfs_register_sysctl_table. It returns a pointer
+ * to \a cfs_sysctl_table_header_t (i.e. the handle of this sysctl
+ * table) that may be used later as argument for
+ * \a cfs_unregister_sysctl_table to remove given sysctl table from
+ * the system.
+ */
+
+/**
+ * \defgroup libcfs_lwt Light-weight tracing
+ *
+ * Support for temporary event tracing with minimal Heisenberg effect
+ *
+ * This facility was designed to allow developers to trace arbitrary
+ * events of interest minimizing the impact of tracing itself on
+ * behavior of the system.
+ *
+ * To use this facility, a developer should add \a LWT_EVENT probes
+ * into some parts of code that need tracing and recompile Lustre
+ * changing "#define LWT_SUPPORT  0" to "#define LWT_SUPPORT  1" in
+ * platform-specific libcfs headers. Then, the developer may use LWT
+ * ioctl-s to enable LWT, gather events and observe them:
+ * - IOC_LIBCFS_LWT_CONTROL
+ * - IOC_LIBCFS_LWT_SNAPSHOT
+ * - IOC_LIBCFS_LWT_LOOKUP_STRING.
+
+ * This facility is meant for finding specific problems and should
+ * never stay in production code.
+ */
+
+/**
+ * \defgroup libcfs_watchdog Watchdogs
+ *
+ * Watchdog facility of Libcfs.
+ *
+ * Libcfs watchdog is a device that allows caller to control whether some
+ * action is performed in timely manner. A consumer interested in
+ * such a control usually creates a watchdog at initialization time,
+ * enables it before start to perform an action specifying \a timeout for
+ * given action, then disables watchdog when processing finished. If
+ * action takes more than \a timeout seconds, Libcfs triggers watchdog
+ * calling consumer-defined callback and dumps lustre logs to a
+ * file. It's responsibility of caller to "touch" or disable watchdog
+ * when the action completed to avoid false triggering.
+ *
+ * If consumer didn't specify callback, the default one would be
+ * called after triggering watchdog. It prints out stack-trace of task
+ * that created watchdog (if possible). The consumer may reuse
+ * existing watchdog without recreation by means of re-enabling
+ * ("touching") it. The consumer usually deletes the watchdog at
+ * finalization time.
+ *
+ * NOTE: Libcfs provides watchdog facility only for in-kernel users.
+ *
+ * Libcfs watchdog is represented by lc_watchdog structure which
+ * implementation is hidden from consumer in watchdog.c. The consumer
+ * uses a pointer to libcfs watchdog as the handle for all operations
+ * on watchdog:
+ * - lc_watchdog_add - creating watchdog
+ * - lc_watchdog_touch - enabling watchdog
+ * - lc_watchdog_disable - disabling watchdog
+ * - lc_watchdog_delete - destroying watchdog
+ *
+ * Internally, libcfs watchdog may be in one of three states:
+ * - ENABLED - Libcfs armed watchdog and ready to trigger it if
+ * consumer won't touch or disable it before \a timeout expires
+ * - DISABLED - watchdog is turned off, Libcfs doesn't pay attention
+ * to it anymore
+ * - EXPIRED - watchdog was triggered, lustre logs dumped, callback
+ * called, Libcfs doesn't pay attention to this watchdog anymore.
+ *
+ * State transitions happen as follows:
+ * - upon creation (lc_watchdog_add), watchdog is in DISABLED state
+ * - being in DISABLED or EXPIRED state, the state is switched to
+ * ENABLED on lc_watchdog_touch call.
+ * - being in ENABLED or EXPIRED state, the state is switched to
+ * DISABLED on lc_watchdog_disable call.
+ * - being in ENABLED state, the state is switched to EXPIRED if
+ * \a timeout lapsed.
+ */
+
+/** \addtogroup libcfs_misc
+ * @{ */
 #if !__GNUC__
 #define __attribute__(x)
 #endif
@@ -56,6 +207,10 @@
 #include "curproc.h"
 
 #ifndef offsetof
+/**
+ * Calculates offset (in bytes) of field \a memb of type (usually
+ *  struct) \a typ counting from the beginning of \a typ.
+ */
 # define offsetof(typ,memb) ((long)(long_ptr_t)((char *)&(((typ *)0)->memb)))
 #endif
 
@@ -64,8 +219,11 @@
 #endif
 
 #if !defined(container_of)
-/* given a pointer @ptr to the field @member embedded into type (usually
- * struct) @type, return pointer to the embedding instance of @type. */
+/**
+ *  Given a pointer \a ptr to the field \a member embedded into type
+ *  (usually struct) \a type, returns pointer to the embedding
+ *  instance of \a type.
+ */
 #define container_of(ptr, type, member) \
         ((type *)((char *)(ptr)-(char *)(&((type *)0)->member)))
 #endif
@@ -75,11 +233,12 @@ static inline int __is_po2(unsigned long
         return !(val & (val - 1));
 }
 
+/** \return TRUE if and only if \a val is power of two */
 #define IS_PO2(val) __is_po2((unsigned long long)(val))
 
 #define LOWEST_BIT_SET(x)       ((x) & ~((x) - 1))
 
-/*
+/**
  * Lustre Error Checksum: calculates checksum
  * of Hex number by XORing each bit.
  */
@@ -87,7 +246,7 @@ static inline int __is_po2(unsigned long
                            ((hexnum) >> 8 & 0xf))
 
 
-/*
+/**
  * Some (nomina odiosa sunt) platforms define NULL as naked 0. This confuses
  * Lustre RETURN(NULL) macro.
  */
@@ -97,14 +256,72 @@ static inline int __is_po2(unsigned long
 
 #define NULL ((void *)0)
 
+/** Well-known LNET PID (of lnet_pid_t type) that all Lustre servers share */
 #define LUSTRE_SRV_LNET_PID      LUSTRE_LNET_PID
 
+/** @} libcfs_misc */
+
+/**
+ * \defgroup libcfs_net Network API
+ *
+ * Generic API for IP and TCP/IP.
+ *
+ * This API consists of three parts:
+ *
+ * 1. Enumerating and querying IP interfaces
+ *
+ * The only consumers of this part are LNDs who need to bind its
+ * instance to IP-address having IP-interface name externally
+ * provided. The API allows to build the table of names of all known
+ * interfaces (what we can see in 'ifconfig -a' output) and query
+ * every particular interface (specified by name) for its properties
+ * like IP-address, net-mask, etc.
+ *
+ * 2. Manipulating TCP/IP sockets
+ *
+ * This part abstracts some operations on TCP/IP sockets allowing LNET
+ * to implement connection establishment routines in generic way
+ * (i.e. hiding platform-specific details). Operations includes:
+ * - listen/connect/accept
+ * - setting and querying socket properties (namely, RX and TX socket
+ * buffer sizes)
+ * - read/write data from/to socket
+ * - release socket (no creation op provided, because listen/connect
+ * creates it internally)
+ *
+ * NOTE: read/write ops are blocking. LNET uses them only during
+ * connection establishment phase (including hello handshake). For
+ * subsequent data transfers, it employs more subtle,
+ * platform-specific techniques. An exception is usocklnd that enables
+ * non-blocking explicitly with libcfs_fcntl_nonblock and uses
+ * scatter-gather versions of libcfs read/write ops for all purposes.
+ *
+ * 3. A few auxiliary macros for easy printing IP-addresses: NIPQUAD and HIPQUAD
+ *
+ * NIPQUAD is for IP-addresses stored in network byte-order, HIPQUAD -
+ * for host byte-order. Typical usage is:
+ *
+ * CDEBUG(D_NET, "ip: %u.%u.%u.%u\n", HIPQUAD(ipaddr)); // __u32 ipaddr;
+ *
+ * @{
+ * \defgroup libcfs_kernel_net Network API (kernel-space)
+ *
+ * See \ref libcfs_net for generic description.
+ */
+/** @} libcfs_net */
+
 #ifdef __KERNEL__
 
 #include <libcfs/list.h>
 
-/* for_each_possible_cpu is defined newly, the former is
- * for_each_cpu(eg. sles9 and sles10) b=15878 */
+/*
+ * for_each_possible_cpu is defined newly, the former is
+ * for_each_cpu(eg. sles9 and sles10) b=15878
+ *
+ * XXX Maxim: it's too weird to have this code here, in generic
+ * libcfs.h; platform-specific parts of libcfs should be responsible
+ * for providing proper for_each_possible_cpu macro
+ */
 #ifndef for_each_possible_cpu
 # ifdef for_each_cpu
 #  define for_each_possible_cpu(cpu) for_each_cpu(cpu)
@@ -113,10 +330,22 @@ static inline int __is_po2(unsigned long
 # endif
 #endif
 
-/* libcfs tcpip */
+/** \addtogroup libcfs_net
+ * @{
+ * \addtogroup libcfs_kernel_net
+ * @{ */
+
+/* enumerating and querying IP-interfaces */
 int libcfs_ipif_query(char *name, int *up, __u32 *ip, __u32 *mask);
 int libcfs_ipif_enumerate(char ***names);
 void libcfs_ipif_free_enumeration(char **names, int n);
+
+/* XXX Maxim: All APIs below excepting getbuf/setbuf are used in
+ * acceptor.c that has one implementation both for kernel and
+ * user-space. So, these APIs below should be factored out of
+ * #ifdef __KERNEL__ ... #endif. They are generic. */
+
+/* libcfs tcpip */
 int libcfs_sock_listen(cfs_socket_t **sockp, __u32 ip, int port, int backlog);
 int libcfs_sock_accept(cfs_socket_t **newsockp, cfs_socket_t *sock);
 void libcfs_sock_abort_accept(cfs_socket_t *sock);
@@ -129,110 +358,331 @@ int libcfs_sock_getaddr(cfs_socket_t *so
 int libcfs_sock_write(cfs_socket_t *sock, void *buffer, int nob, int timeout);
 int libcfs_sock_read(cfs_socket_t *sock, void *buffer, int nob, int timeout);
 void libcfs_sock_release(cfs_socket_t *sock);
+/** @} libcfs_kernel_net */
+/** @} libcfs_net */
 
-/* libcfs watchdogs */
+/** \addtogroup libcfs_watchdog
+ * @{ */
 struct lc_watchdog;
 
-/* Add a watchdog which fires after "time" milliseconds of delay.  You have to
- * touch it once to enable it. */
+/** Add a watchdog which fires after "time" milliseconds of delay.
+ * You have to touch it once to enable it. */
 struct lc_watchdog *lc_watchdog_add(int time,
                                     void (*cb)(pid_t pid, void *),
                                     void *data);
 
-/* Enables a watchdog and resets its timer. */
+/** Enables a watchdog and resets its timer. */
 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout);
 #define GET_TIMEOUT(svc) (max_t(int, obd_timeout,                       \
                           AT_OFF ? 0 : at_get(&svc->srv_at_estimate)) * \
                           svc->srv_watchdog_factor)
 
-/* Disable a watchdog; touch it to restart it. */
+/** Disable a watchdog; touch it to restart it. */
 void lc_watchdog_disable(struct lc_watchdog *lcw);
 
-/* Clean up the watchdog */
+/** Clean up the watchdog */
 void lc_watchdog_delete(struct lc_watchdog *lcw);
 
-/* Dump a debug log */
+/** Dump a debug log. It's the default callback to call when
+ * watchdog is triggered */
 void lc_watchdog_dumplog(pid_t pid, void *data);
 
+/** @} libcfs_watchdog */
+
 #endif /* __KERNEL__ */
 
-/* need both kernel and user-land acceptor */
+/*
+ * need both kernel and user-land acceptor
+ *
+ * XXX Maxim: These defines have nothing to do with libcfs API. Only
+ * LNET acceptor, ksocklnd and usocklnd need them. To be moved to
+ * generic lnet.h
+ */
 #define LNET_ACCEPTOR_MIN_RESERVED_PORT    512
 #define LNET_ACCEPTOR_MAX_RESERVED_PORT    1023
 
-/*
- * libcfs pseudo device operations
+/**
+ * \defgroup libcfs_ioctl Pseudo devices and ioctls support
+ * Generic API to support ioctls and manipulating pseudo-devices.
+ *
+ * Libcfs provides two levels of support for ioctls:
+ *
+ * 1. Ioctls that land on /dev/lnet device
+ *
+ * Libcfs creates and maintains /dev/lnet internally providing very
+ * simple API for consumers who need to handle ioctls coming via
+ * /dev/lnet. A consumer only needs to register its ioctl-handlers of
+ * \a libcfs_ioctl_handler type with libcfs_register_ioctl call on
+ * startup and unregister it with libcfs_deregister_ioctl on shutdown.
+ *
+ * Libcfs maintains the list of all registered ioctl-handlers using
+ * global \a ioctl_list as list-head. Whenever a user issues an ioctl on
+ * /dev/lnet, Libcfs firstly checks its \a cmd for values that can be
+ * handled internally (like IOC_LIBCFS_MARK_DEBUG, IOC_LIBCFS_CLEAR_DEBUG
+ * and some others), and, if it cannot, iterates over \a ioctl_list
+ * calling registered handlers one by one until some handler recognizes
+ * \a cmd returning rc != -EINVAL.
  *
- * struct cfs_psdev_t and
- * cfs_psdev_register() and
- * cfs_psdev_deregister() are declared in
- * libcfs/<os>/<os>-prim.h
+ * Libcfs also provides generic way to pass ioctl data to and from
+ * user-space calling libcfs_ioctl_getdata when an ioctl arrives and
+ * libcfs_ioctl_popdata when ioctl handling is complete. This requires
+ * using \a libcfs_ioctl_data structure as ioctl data both on
+ * user-space side issuing ioctls and kernel-space ioctl-handlers. So,
+ * both user-space and kernel handlers must share libcfs conventions
+ * of \a libcfs_ioctl_data content. See implementation of
+ * libcfs_ioctl_is_invalid, libcfs_ioctl_getdata and
+ * libcfs_ioctl_popdata for details.
  *
- * It's just draft now.
+ * Note, that libcfs_ioctl.h defines all ioctls handled via /dev/lnet
+ * device. The rationale is avoidance of overlap that could happen if
+ * every consumer defined \a cmd values on its own.
+ *
+ * 2. Any other ioctls
+ *
+ * Supporting ioctls for devices other than /dev/lnet requires
+ * creation of such devices first of all. Platform-specific libcfs
+ * files defines cfs_psdev_t type as an entity that can be registered
+ * in kernel to create /dev/something visible from user-space. They
+ * also provides register/unregister primitives to deal with
+ * cfs_psdev_t:
+ * - cfs_psdev_register
+ * - cfs_psdev_deregister
+ *
+ * It's responsibility of consumer of Libcfs to instantiate a sample
+ * of cfs_psdev_t, register and unregister it with the primitives
+ * above. Usually, cfs_psdev_t bears the name of pseudo-device (what
+ * will appear in /dev subtree) and the set of callbacks to be
+ * called by kernel whenever an user issues open/ioctl/close syscall
+ * on /dev/name_of_pseudo_device. The implementation of these
+ * callbacks is platform- and consumer-specific but usually they just
+ * call another callbacks - ones defined in \a cfs_psdev_ops structure
+ * correspondent to given pseudo-device. \a cfs_psdev_ops is not
+ * platform-specific and consumer may define it in common c-files.
+ *
+ * \a cfs_psdev_ops bears open/ioctl/close callbacks to be called from
+ * platform-specific consumer code whenever correspondent events
+ * happen. open/close callbacks may be implemented as bare
+ * PORTAL_MODULE_USE/PORTAL_MODULE_UNUSE (to avoid modunload in the
+ * middle of operation) if consumer doesn't need user-state
+ * info. ioctl callback is usually big switch over all allowed \a cmd
+ * values.
+ *
+ * The main consumer of facility explained above is Libcfs
+ * itself. Libcfs uses the same technique to create and maintain
+ * /dev/lnet as it provides for external consumers. Note that unlike
+ * to other consumers, Libcfs does need user-state info preserved
+ * across syscalls to handle ioctl(IOC_LIBCFS_MEMHOG)
+ * correctly. User-state feature works for this purpose as follows:
+ *
+ * - platform-specific libcfs open callback provides some storage to store
+ * a pointer to user-state info and passes its address as the second
+ * argument to common libcfs open callback. This storage resides in
+ * kernel, it is bound to open file descriptor and preserved across
+ * syscalls by kernel.
+ * - common libcfs open callback allocates user-state info of type
+ * \a libcfs_device_userstate and save its address where the second
+ * argument specifies.
+ * - when ioctl(IOC_LIBCFS_MEMHOG) comes, platform-specific libcfs
+ * ioctl callback creates an instance of \a cfs_psdev_file structure
+ * and copies the address of user-state info from the in-kernel
+ * storage to this instance. Then, it calls libcfs common ioctl
+ * callback passing the address of newly created and filled \a cfs_psdev_t
+ * as the first argument.
+ * - common libcfs ioctl callback interprets the first argument as
+ * pointer to \a libcfs_device_userstate structure (i.e. user-state
+ * info) and saves number of allocated memhog pages and pointer to the
+ * root memhog page int its fields
+ * - even if user-space process issuing ioctl(IOC_LIBCFS_MEMHOG)
+ * aborted, the kernel guarantees that file descriptor of /dev/lnet
+ * will be closed and libcfs platform-specific close callback
+ * called. The latter gets the address of user-state info from
+ * in-kernel storage and passes it as the second argument to common
+ * libcfs close callback.
+ * - common libcfs close callback interprets the second argument as
+ * pointer to \a libcfs_device_userstate structure (i.e. user-state
+ * info) and releases memhog pages according to the values stored in
+ * the fields of \a libcfs_device_userstate.
+ *
+ * @{
  */
 
+/**
+ * libcfs pseudo device file descriptor
+ *
+ * Libcfs ensures that cfs_psdev_ops::ioctl will be called with
+ * pointer to \a cfs_psdev_file whose \a private_data field is initialized
+ * with the same pointer that cfs_psdev_ops::open previously stored
+ * dereferencing its second argument.
+ */
 struct cfs_psdev_file {
+        /** not used */
         unsigned long   off;
+        /** The address of storage persistent across syscalls.
+         * \a cfs_psdev_ops methods may store address of user-state
+         * info in that storage */
         void            *private_data;
+        /** reserved for future use */
         unsigned long   reserved1;
+        /** reserved for future use */
         unsigned long   reserved2;
 };
 
+/**
+ * libcfs pseudo device operations
+ *
+ * Callbacks registered in this structure are being called while
+ * processing file operation on /dev/something device as follows:
+ * user-space open/ioctl/close syscall -->
+ *    kernel system handler of open/ioctl/close -->
+ *        libcfs platform-specific open/ioctl/close callback
+ *        registered in \a cfs_psdev_t -->
+ *            open/ioctl/close callback registered in \a cfs_psdev_ops.
+ */
 struct cfs_psdev_ops {
+        /** open callback to be called by platform-specific libcfs */
         int (*p_open)(unsigned long, void *);
+        /** close callback to be called by platform-specific libcfs */
         int (*p_close)(unsigned long, void *);
+        /** not used */
         int (*p_read)(struct cfs_psdev_file *, char *, unsigned long);
+        /** not used */
         int (*p_write)(struct cfs_psdev_file *, char *, unsigned long);
+        /** ioctl callback to be called by platform-specific libcfs */
         int (*p_ioctl)(struct cfs_psdev_file *, unsigned long, void *);
 };
 
-/*
- * Universal memory allocator API
+/** @} libcfs_ioctl */
+
+/**
+ * \defgroup libcfs_mem Memory Allocations
+ * Generic API for memory allocations.
+ *
+ * The API includes:
+ *
+ * - cfs_page_t as a generic wrapper around platform-specific native
+ * page structure along with routines to alloc, free, map, unmap and
+ * get its address.
+ * - general memory allocation routines to alloc and free a memory
+ * buffer of given size.
+ * - cfs_mem_cache_t - memory cache structure (SLAB allocator) along
+ * with routines to alloc, free, create and destroy it.
+ * - the set of universal flags to control allocator behavior and the
+ * type of memory allocated (like whether allocator can block, whether
+ * the kernel can start disk I/O for allocated buffer, etc.)
+ *
+ * Usage of libcfs page:
+ * - a page can be allocated with cfs_alloc_page
+ * - an array of pages can be allocated with cfs_alloc_pages
+ * - allocated page should be returned after use with cfs_free_page
+ * - allocated array of pages should be returned after use with cfs_free_pages
+ * - an allocated page can be mapped (i.e. associated with vaddr) with
+ * cfs_kmap
+ * - if page is mapped, the caller can get its vaddr with
+ * cfs_page_address
+ * - mapped page should be unmapped when its vaddr is not in use anymore
+ * with cfs_kunmap
+ *
+ * Usage of libcfs memory cache:
+ * - a cache can be created with cfs_mem_cache_create
+ * - created cache should be released after use with cfs_mem_cache_destroy
+ * - if a cache is created, the caller can allocate an object from
+ * this cache with cfs_mem_cache_alloc and return it after use with
+ * cfs_mem_cache_free. The object is just a chunk of virtual memory of
+ * given size specified by its vaddr.
+ * - the caller can get an imprecise hint whether given vaddr was
+ * allocated in the cache with cfs_mem_is_in_cache.
+ *
+ * @{
+ */
+
+/**
+ * Universal memory allocator flags
  */
 enum cfs_alloc_flags {
-        /* allocation is not allowed to block */
+        /** allocation is not allowed to block */
         CFS_ALLOC_ATOMIC = 0x1,
-        /* allocation is allowed to block */
+        /** allocation is allowed to block */
         CFS_ALLOC_WAIT   = 0x2,
-        /* allocation should return zeroed memory */
+        /** allocation should return zeroed memory */
         CFS_ALLOC_ZERO   = 0x4,
-        /* allocation is allowed to call file-system code to free/clean
+        /** allocation is allowed to call file-system code to free/clean
          * memory */
         CFS_ALLOC_FS     = 0x8,
-        /* allocation is allowed to do io to free/clean memory */
+        /** allocation is allowed to do io to free/clean memory */
         CFS_ALLOC_IO     = 0x10,
-        /* don't report allocation failure to the console */
+        /** don't report allocation failure to the console */
         CFS_ALLOC_NOWARN = 0x20,
-        /* standard allocator flag combination */
+        /** standard allocator flag combination */
         CFS_ALLOC_STD    = CFS_ALLOC_FS | CFS_ALLOC_IO,
+        /** flag combination for user process memory */
         CFS_ALLOC_USER   = CFS_ALLOC_WAIT | CFS_ALLOC_FS | CFS_ALLOC_IO,
 };
 
-/* flags for cfs_page_alloc() in addition to enum cfs_alloc_flags */
+/** flags for cfs_page_alloc() in addition to enum cfs_alloc_flags */
 enum cfs_alloc_page_flags {
-        /* allow to return page beyond KVM. It has to be mapped into KVM by
-         * cfs_page_map(); */
+        /** allow to return page beyond KVM. It has to be mapped into KVM by
+         * cfs_kmap(); */
         CFS_ALLOC_HIGH   = 0x40,
+        /** combines ::CFS_ALLOC_HIGH and ::CFS_ALLOC_USER */
         CFS_ALLOC_HIGHUSER = CFS_ALLOC_WAIT | CFS_ALLOC_FS | CFS_ALLOC_IO | CFS_ALLOC_HIGH,
 };
+/** @} libcfs_mem */
 
-/*
- * Drop into debugger, if possible. Implementation is provided by platform.
- */
+/** \addtogroup libcfs_misc
+ * @{ */
 
+/** Drop into debugger, if possible. Implementation is provided by platform. */
 void cfs_enter_debugger(void);
 
-/*
- * Defined by platform
+/**
+ * Convert server error code to client format.
+ * Error codes are from Linux errno.h.
+ */
+int convert_server_error(__u64 ecode);
+
+/** convert <fcntl.h> flag from client to server. */
+int convert_client_oflag(int cflag, int *result);
+
+/** @} libcfs_misc */
+
+/**
+ * \defgroup libcfs_signal Signal and daemonization
+ * Generic API for unix signals and daemonization (defined by platform)
+ *
+ * Daemonization assumes detaching from controlling terminal, blocking
+ * all signals, releasing file-system resources (like current working
+ * dir) and re-parenting to 'init' unix process.
+ *
+ * NOTE: Both cfs_daemonize and cfs_daemonize_ctxt daemonize caller,
+ * but consumer must use the latter if she is going to change task
+ * parameters related to file-system (like current working dir)
+ * afterwards. Otherwise, if given thread has nothing to do with
+ * file-system part of task, cfs_daemonize is enough.
+ *
+ * Signal-related primitives manipulates 'blocked signals' mask and
+ * 'pending' flag. 'blocked signals' mask defines which signals to
+ * block (i.e. don't pass to given task). The mask is represented by
+ * cfs_sigset_t type. 'pending' flag indicates that some signal is
+ * pending for given task (i.e. waits for delivery).
+ * @{
  */
+/** daemonize */
 void cfs_daemonize(char *str);
+/** daemonize and clone file-system part of task */
 int cfs_daemonize_ctxt(char *str);
+/** inquire current 'blocked signals' mask */
 cfs_sigset_t cfs_get_blocked_sigs(void);
+/** block all signals */
 cfs_sigset_t cfs_block_allsigs(void);
+/** block particular signals (\a bits); returns old mask */
 cfs_sigset_t cfs_block_sigs(cfs_sigset_t bits);
+/** restore mask to old value (to be passed as argument) */
 void cfs_restore_sigs(cfs_sigset_t);
+/** check if a signal waits for delivery */
 int cfs_signal_pending(void);
+/** clear 'pending' flag */
 void cfs_clear_sigpending(void);
+
 /*
  * XXX Liang:
  * these macros should be removed in the future,
@@ -242,30 +692,81 @@ void cfs_clear_sigpending(void);
 #define libcfs_daemonize(s)     cfs_daemonize(s)
 #define cfs_sigmask_lock(f)     do { f= 0; } while (0)
 #define cfs_sigmask_unlock(f)   do { f= 0; } while (0)
+/** @} libcfs_signal */
 
-int convert_server_error(__u64 ecode);
-int convert_client_oflag(int cflag, int *result);
-
-/*
- * Stack-tracing filling.
+/** \defgroup libcfs_debug Debugging
+ * Various subsystems of Libcfs to alleviate debugging
+ * @{
+ * \defgroup libcfs_stack_trace Stack-tracing
+ * Generic API for filling and extracting stack-traces
+ * @{
  */
 
-/*
- * Platform-dependent data-type to hold stack frames.
- */
+/* XXX Maxim: Nobody uses this API for now. Should we eliminate this
+ * subsystem as obsoleted? */
+
+/** Platform-dependent data-type to hold stack frames. */
 struct cfs_stack_trace;
 
-/*
- * Fill @trace with current back-trace.
- */
+/** Fill \a trace with current back-trace. */
 void cfs_stack_trace_fill(struct cfs_stack_trace *trace);
 
-/*
- * Return instruction pointer for frame @frame_no. NULL if @frame_no is
+/**
+ * Extract info (instruction pointer) for certain frame.
+ * \param trace stack-trace
+ * \param frame_no which frame we're querying for
+ * \return instruction pointer for frame \a frame_no. NULL if \a frame_no is
  * invalid.
  */
 void *cfs_stack_trace_frame(struct cfs_stack_trace *trace, int frame_no);
 
+/** @} libcfs_stack_trace */
+/** @} libcfs_debug */
+
+/**
+ * \defgroup libcfs_fs File operations for kernel users
+ * Generic API for manipulating files in kernel
+ *
+ * At present, the API consists of four independent parts:
+ *
+ * 1) File type and routines to read/write a file:
+ * - cfs_file_t
+ * - cfs_filp_open
+ * - cfs_filp_close
+ * - cfs_filp_read
+ * - cfs_user_write
+ * - cfs_filp_fsync
+ * - cfs_get_file
+ * - cfs_put_file
+ * - cfs_file_count
+ * - cfs_filp_size
+ * - cfs_filp_poff
+ * - cfs_user_write
+ *
+ * 2) File-lock type and related functions:
+ * - cfs_flock_t
+ * - cfs_flock_type
+ * - cfs_flock_set_type
+ * - cfs_flock_pid
+ * - cfs_flock_set_pid
+ * - cfs_flock_start
+ * - cfs_flock_set_start
+ * - cfs_flock_end
+ * - cfs_flock_set_end
+ *
+ * 3) Types and macros to work with directories:
+ * - cfs_dentry_t
+ * - cfs_dirent_t
+ * - CFS_IFTODT
+ * - CFS_DTTOIF
+ *
+ * 4) Universal open flags and conversion routines (currently unused):
+ * - CFS_O_ACCMODE, CFS_O_CREAT, etc.
+ * - cfs_oflags2univ
+ * - cfs_univ2oflags
+ * @{
+ */
+
 /*
  * Universal open flags.
  */
@@ -285,11 +786,13 @@ void *cfs_stack_trace_frame(struct cfs_s
 #define CFS_O_NOFOLLOW          0400000
 #define CFS_O_NOATIME           01000000
 
-/* convert local open flags to universal open flags */
+/** convert local open flags to universal open flags */
 int cfs_oflags2univ(int flags);
-/* convert universal open flags to local open flags */
+/** convert universal open flags to local open flags */
 int cfs_univ2oflags(int flags);
 
+/** @} libcfs_fs */
+
 #include <libcfs/libcfs_debug.h>
 #include <libcfs/libcfs_private.h>
 #include <libcfs/libcfs_ioctl.h>
@@ -299,7 +802,10 @@ int cfs_univ2oflags(int flags);
 #include <libcfs/libcfs_kernelcomm.h>
 #include <libcfs/libcfs_hash.h>
 
-/* container_of depends on "likely" which is defined in libcfs_private.h */
+/** \addtogroup libcfs_misc
+ * @{ */
+
+/** container_of depends on "likely" which is defined in libcfs_private.h */
 static inline void *__container_of(void *ptr, unsigned long shift)
 {
         if (unlikely(IS_ERR(ptr) || ptr == NULL))
@@ -308,9 +814,12 @@ static inline void *__container_of(void 
                 return (char *)ptr - shift;
 }
 
+/** the same as ::container_of, but also checks validity of \a ptr */
 #define container_of0(ptr, type, member) \
         ((type *)__container_of((void *)(ptr), offsetof(type, member)))
+/** @} libcfs_misc */
 
 #define _LIBCFS_H
 
+/** @} libcfs */
 #endif /* _LIBCFS_H */
Index: libcfs/include/libcfs/libcfs_debug.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_debug.h,v
retrieving revision 1.12
diff -u -p -r1.12 libcfs_debug.h
--- libcfs/include/libcfs/libcfs_debug.h	15 Sep 2009 17:52:18 -0000	1.12
+++ libcfs/include/libcfs/libcfs_debug.h	1 Jan 2010 20:03:31 -0000
@@ -42,6 +42,11 @@
 #ifndef __LIBCFS_DEBUG_H__
 #define __LIBCFS_DEBUG_H__
 
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_debug
+ * @{ */
+
 /*
  *  Debugging
  */
@@ -68,6 +73,75 @@ extern unsigned int libcfs_catastrophe;
 extern unsigned int libcfs_panic_on_lbug;
 
 /**
+ * \defgroup libcfs_debug_print Debug printing
+ * Generic API to print debug messages
+ *
+ * The API for printing debug messages includes the following macros:
+ * - CDEBUG
+ * - CWARN
+ * - CERROR
+ * - LCONSOLE
+ * - LCONSOLE_INFO
+ * - LCONSOLE_WARN
+ * - LCONSOLE_ERROR_MSG
+ * - LCONSOLE_ERROR
+ * - LCONSOLE_EMERG
+ *
+ * These macros may print a message to internal log buffers provided
+ * by tracefile subsystem when used in kernel or to a file which name
+ * is specified in LIBLUSTRE_DEBUG_FILE environment variable in
+ * user-space case. They also may print out the message on the
+ * console. Making decision about printing particular message depends
+ * on the notions of \a mask and \a subsystem.
+ *
+ * \a subsystem represents a logical part of Lustre. For example,
+ * S_LNET for LNET subsystem, S_LDLM for LDLM, etc. Internally, \a
+ * subsystem is a combination of bit-flags S_MDC, S_MDS, and many
+ * others (see "Debugging subsystems" in libcfs_debug.h). The caller
+ * must define c-preprocessor variable DEBUG_SUBSYSTEM with desired
+ * \a subsystem before calling any debug print macro. Typically, such
+ * a define comes in the beginning of each c-file.
+ *
+ * \a mask represents an "area of interest" like warnings, errors,
+ * ioctl-related events etc. Internally, \a mask is a combination of
+ * bit-flags D_TRACE, D_INODE, and many others (see "Debugging masks"
+ * in libcfs_debug.h). In case of CDEBUG and LCONSOLE macros, the
+ * caller specifies \a mask explicitly, as the first argument of
+ * macro. In other cases, \a masks are predefined like D_WARNING for
+ * CWARN macro, D_ERROR for CERROR macro, etc.
+ *
+ * D_CONSOLE flag is reserved for indicating the fact that given debug
+ * print should appears on the console. LCONSOLE macro adds this flag
+ * to caller-supplied \a mask to ensure printing on console.
+ 
+ * Typically, all macros from the list above (excepting CDEBUG) print
+ * the message on console as well as in internal memory buffers. This
+ * behavior may be changed by tuning global variable \a libcfs_printk.
+ * This variable is a \a mask consisting of the flags for which
+ * console printing is desired.
+ *
+ * As for CDEBUG, Libcfs firstly makes decision whether given debug
+ * print should be printed at all. The decision is  based on \a mask
+ * and \a subsystem (see implementation of cdebug_show). If the
+ * decision is positive, the debug print always goes into internal log
+ * buffers and optionally may appear on console. Console print happens
+ * if the caller used \a mask (as its first argument) containing any
+ * flags from libcfs_printk.
+ *
+ * All of debug print APIs excepting CDEBUG and LCONSOLE are
+ * rate-limited. This means that Libcfs skips console printing for
+ * them if the same macro is invoked from the same line of code too
+ * often. The delay varies from \a libcfs_console_min_delay to
+ * \a libcfs_console_max_delay adaptively. User can disable this feature
+ * by setting global variable \a libcfs_console_ratelimit to zero.
+ *
+ * NOTE: In user-space case, console printing and rate-limiting are
+ * only available when header-file named "catamount/data.h" is present
+ * while building Lustre.
+ * @{
+ */
+
+/**
  * Format for debug message headers
  */
 struct ptldebug_header {
@@ -239,6 +313,8 @@ do {                                    
 
 #define LCONSOLE_EMERG(format, ...) CDEBUG(D_CONSOLE | D_EMERG, format, ## __VA_ARGS__)
 
+/** @} libcfs_debug_print */
+
 #ifdef CDEBUG_ENABLED
 
 #define GOTO(label, rc)                                                 \
@@ -304,6 +380,9 @@ do {                                    
         return;                                                         \
 } while (0)
 
+/** \addtogroup libcfs_debug_print
+ * @{ */
+
 struct libcfs_debug_msg_data {
         cfs_debug_limit_state_t *msg_cdls;
         int                      msg_subsys;
@@ -350,14 +429,6 @@ extern int libcfs_debug_vmsg2(cfs_debug_
                                  (file), (func), (line), fmt, ## __VA_ARGS__);\
 } while(0)
 
-extern void libcfs_assertion_failed(const char *expr, const char *file,
-                                    const char *fn, const int line);
-
-/* one more external symbol that tracefile provides: */
-extern int trace_copyout_string(char *usr_buffer, int usr_buffer_nob,
-                                const char *knl_buffer, char *append);
-
-
 #if defined(HAVE_BGL_SUPPORT)
 #define DEBUG_FILE_PATH_DEFAULT "/bgl/ion/tmp/lustre-log"
 #elif defined(__arch_um__)
@@ -368,4 +439,16 @@ extern int trace_copyout_string(char *us
 #define DEBUG_FILE_PATH_DEFAULT "/tmp/lustre-log"
 #endif
 
+/** @} libcfs_debug_print */
+
+extern void libcfs_assertion_failed(const char *expr, const char *file,
+                                    const char *fn, const int line);
+
+/* one more external symbol that tracefile provides: */
+extern int trace_copyout_string(char *usr_buffer, int usr_buffer_nob,
+                                const char *knl_buffer, char *append);
+
+/** @} libcfs_debug */
+/** @} libcfs */
+
 #endif	/* __LIBCFS_DEBUG_H__ */
Index: libcfs/include/libcfs/libcfs_hash.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_hash.h,v
retrieving revision 1.4
diff -u -p -r1.4 libcfs_hash.h
--- libcfs/include/libcfs/libcfs_hash.h	1 Dec 2009 02:23:43 -0000	1.4
+++ libcfs/include/libcfs/libcfs_hash.h	1 Jan 2010 20:03:32 -0000
@@ -41,6 +41,45 @@
 
 #ifndef __LIBCFS_HASH_H__
 #define __LIBCFS_HASH_H__
+
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup hash_table Hash table
+ * An abstraction layer over 'hash table' technique
+ *
+ * Libcfs Hashing provides an abstraction layer over well-known 'hash
+ * table' technique that uses a hash function to efficiently map
+ * certain identifiers or keys to associated values. The key point is
+ * maintaining an array of 'buckets' where each bucket chains the list
+ * of items stored in hash table and using some hash function 'f' to
+ * locate proper slot in the array like this: index = f(key, array_length).
+ *
+ * A consumer should provide the following prerequisites to use this facility:
+ * - a storage for hlist_node for each item (usually embedded into
+ * item's data structure)
+ * - some descriptive name for the given hash table
+ * - initial and maximum allowed hash table sizes
+ * - a function calculating key for given item
+ * - hash function suitable for given distribution of keys
+ * - 'compare' function determining whether given item has given key
+ * - get/put functions converting pointer to hlist_node of item to the
+ * pointer to the whole item (can also be used to update reference
+ * counter of item)
+ * - optional exit function that will be called for every item after
+ * its removal from hash table while its destruction.
+ *
+ * Having all bits listed above provided, libcfs hash subsystem allows:
+ * - create and destroy hash table
+ * - add, delete and lookup entries in hash table
+ * - rehash item if its key was changed
+ * - print out internal hash table parameters for debug purposes
+ * - dynamic rehashing (i.e. rebuilding hash table to keep load factor
+ * low)
+ * @{
+ */
+
 /*
  * Knuth recommends primes in approximately golden ratio to the maximum
  * integer representable by a machine word for multiplicative hashing.
@@ -51,9 +90,9 @@
  * them can use shifts and additions instead of multiplications for
  * machines where multiplications are slow.
  */
-/* 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */
+/** 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */
 #define CFS_GOLDEN_RATIO_PRIME_32 0x9e370001UL
-/*  2^63 + 2^61 - 2^57 + 2^54 - 2^51 - 2^18 + 1 */
+/**  2^63 + 2^61 - 2^57 + 2^54 - 2^51 - 2^18 + 1 */
 #define CFS_GOLDEN_RATIO_PRIME_64 0x9e37fffffffc0001ULL
 
 /*
@@ -70,10 +109,10 @@
    (C) 2002 William Lee Irwin III, IBM */
 
 #if BITS_PER_LONG == 32
-/* 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */
+/** 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */
 #define CFS_GOLDEN_RATIO_PRIME          CFS_GOLDEN_RATIO_PRIME_32
 #elif BITS_PER_LONG == 64
-/*  2^63 + 2^61 - 2^57 + 2^54 - 2^51 - 2^18 + 1 */
+/**  2^63 + 2^61 - 2^57 + 2^54 - 2^51 - 2^18 + 1 */
 #define CFS_GOLDEN_RATIO_PRIME          CFS_GOLDEN_RATIO_PRIME_64
 #else
 #error Define CFS_GOLDEN_RATIO_PRIME for your wordsize.
@@ -118,41 +157,86 @@ static inline unsigned long hash_ptr(voi
 
 struct cfs_hash_ops;
 
+/**
+ * Represents the chain of items producing the same hash_func value
+ */
 typedef struct cfs_hash_bucket {
-        struct hlist_head           hsb_head;       /* entries list */
-        atomic_t                    hsb_count;      /* current entries */
-        rwlock_t                    hsb_rwlock;     /* cfs_hash_bucket */
+        /** entries list */
+        struct hlist_head           hsb_head;
+        /** # current entries */
+        atomic_t                    hsb_count;
+        /** protects operations on given bucket */
+        rwlock_t                    hsb_rwlock;
 } cfs_hash_bucket_t;
 
 #define CFS_MAX_HASH_NAME 16
 
+/**
+ * Libcfs hash table. Essentially, is array of buckets along with some
+ * bits of information necessary to maintain it.
+ */
 typedef struct cfs_hash {
-        int                         hs_cur_bits;    /* current hash bits */
-        int                         hs_cur_mask;    /* current hash mask */
-        int                         hs_min_bits;    /* min hash bits */
-        int                         hs_max_bits;    /* max hash bits */
-        int                         hs_min_theta;   /* resize min threshold */
-        int                         hs_max_theta;   /* resize max threshold */
-        int                         hs_flags;       /* hash flags */
-        atomic_t                    hs_count;       /* current entries */
-        atomic_t                    hs_rehash_count;/* resize count */
-        struct cfs_hash_bucket    **hs_buckets;     /* hash buckets */
-        struct cfs_hash_ops        *hs_ops;         /* hash operations */
-        rwlock_t                    hs_rwlock;      /* cfs_hash */
+        /** current hash bits */
+        int                         hs_cur_bits;
+        /** current hash mask */
+        int                         hs_cur_mask;
+        /** min hash bits */
+        int                         hs_min_bits;
+        /** max hash bits */
+        int                         hs_max_bits;
+        /** resize min threshold */
+        int                         hs_min_theta;
+        /** resize max threshold */
+        int                         hs_max_theta;
+        /** hash flags */
+        int                         hs_flags;
+        /** # current entries */
+        atomic_t                    hs_count;
+        /** resize count */
+        atomic_t                    hs_rehash_count;
+        /** hash buckets */
+        struct cfs_hash_bucket    **hs_buckets;
+        /** hash operations */
+        struct cfs_hash_ops        *hs_ops;
+        /** protects fields of cfs_hash while rehashing */
+        rwlock_t                    hs_rwlock;
+        /** some descriptive name of this hash */
         char                        hs_name[CFS_MAX_HASH_NAME];
 } cfs_hash_t;
 
+/** Consumer-supplied operations specific for given hash table */
 typedef struct cfs_hash_ops {
+        /** hash function operating on \a key */
         unsigned (*hs_hash)(cfs_hash_t *hs, void *key, unsigned mask);
+        /** calculates \a key for item represented by \a hnode */
         void *   (*hs_key)(struct hlist_node *hnode);
+        /**
+         * checks whether item correspondent to \a hnode has key
+         * equal to \a key
+         */
         int      (*hs_compare)(void *key, struct hlist_node *hnode);
+        /**
+         * Converts \a hnode to item pointer.
+         * Also, hints consumer that the item is in use.
+         */
         void *   (*hs_get)(struct hlist_node *hnode);
+        /**
+         * Converts \a hnode to item pointer.
+         * Also, hints consumer that the item isn't in use anymore.
+         */
         void *   (*hs_put)(struct hlist_node *hnode);
+        /**
+         * optional; will be called for every item after its removal
+         * from hash table while its destruction
+         */
         void     (*hs_exit)(struct hlist_node *hnode);
 } cfs_hash_ops_t;
 
-#define CFS_HASH_DEBUG          0x0001  /* Enable expensive debug checks */
-#define CFS_HASH_REHASH         0x0002  /* Enable dynamic hash resizing */
+/** Enable expensive debug checks */
+#define CFS_HASH_DEBUG          0x0001
+
+/** Enable dynamic hash resizing */
+#define CFS_HASH_REHASH         0x0002
 
 #define CFS_HO(hs)             (hs)->hs_ops
 #define CFS_HOP(hs, op)        (hs)->hs_ops->hs_ ## op
@@ -180,7 +264,12 @@ cfs_hash_key(cfs_hash_t *hs, struct hlis
         return NULL;
 }
 
-/* Returns 1 on a match,
+/**
+ * Check whether the element correspondent to \a hnode has
+ * key equal to \a key
+ *
+ * \return 1 on a match, 0 otherwise.
+ *
  * XXX: This would be better if it returned, -1, 0, or 1 for
  *      <, =, > respectivly.  It could then be used to implement
  *      a CFS_HASH_SORT feature flags which could keep each hash
@@ -239,7 +328,7 @@ cfs_hash_exit(cfs_hash_t *hs, struct hli
                 return CFS_HOP(hs, exit)(hnode);
 }
 
-/* Validate hnode references the correct key */
+/** Validate hnode references the correct key */
 static inline void
 __cfs_hash_key_validate(cfs_hash_t *hs, void *key,
                         struct hlist_node *hnode)
@@ -248,7 +337,7 @@ __cfs_hash_key_validate(cfs_hash_t *hs, 
                 LASSERT(cfs_hash_compare(hs, key, hnode) > 0);
 }
 
-/* Validate hnode is in the correct bucket */
+/** Validate hnode is in the correct bucket */
 static inline void
 __cfs_hash_bucket_validate(cfs_hash_t *hs, cfs_hash_bucket_t *hsb,
                            struct hlist_node *hnode)
@@ -300,25 +389,40 @@ __cfs_hash_bucket_del(cfs_hash_t *hs,
         return cfs_hash_put(hs, hnode);
 }
 
-/* Hash init/cleanup functions */
+/**
+ *\defgroup hash_init_fini Hash init/cleanup functions
+ * @{
+ */
 cfs_hash_t *cfs_hash_create(char *name, unsigned int cur_bits,
                             unsigned int max_bits,
                             cfs_hash_ops_t *ops, int flags);
 void cfs_hash_destroy(cfs_hash_t *hs);
+/** @} */
 
-/* Hash addition functions */
+/**
+ *\defgroup hash_adds Hash addition functions
+ * @{
+ */
 void cfs_hash_add(cfs_hash_t *hs, void *key,
                   struct hlist_node *hnode);
 int cfs_hash_add_unique(cfs_hash_t *hs, void *key,
                         struct hlist_node *hnode);
 void *cfs_hash_findadd_unique(cfs_hash_t *hs, void *key,
                               struct hlist_node *hnode);
+/** @} */
 
-/* Hash deletion functions */
+/**
+ *\defgroup hash_del Hash deletion functions
+ * @{
+ */
 void *cfs_hash_del(cfs_hash_t *hs, void *key, struct hlist_node *hnode);
 void *cfs_hash_del_key(cfs_hash_t *hs, void *key);
+/** @} */
 
-/* Hash lookup/for_each functions */
+/**
+ *\defgroup hash_lookup Hash lookup/for_each functions
+ * @{
+ */
 void *cfs_hash_lookup(cfs_hash_t *hs, void *key);
 typedef void (*cfs_hash_for_each_cb_t)(void *obj, void *data);
 void cfs_hash_for_each(cfs_hash_t *hs, cfs_hash_for_each_cb_t, void *data);
@@ -326,25 +430,30 @@ void cfs_hash_for_each_safe(cfs_hash_t *
 void cfs_hash_for_each_empty(cfs_hash_t *hs, cfs_hash_for_each_cb_t, void *data);
 void cfs_hash_for_each_key(cfs_hash_t *hs, void *key,
                            cfs_hash_for_each_cb_t, void *data);
+/** @} */
 
-/*
+/**
+ *\defgroup hash_rehash Hash rehash functions
+ *
  * Rehash - Theta is calculated to be the average chained
- * hash depth assuming a perfectly uniform hash funcion.
+ * hash depth assuming a perfectly uniform hash function.
+ * @{
  */
 int cfs_hash_rehash(cfs_hash_t *hs, int bits);
 void cfs_hash_rehash_key(cfs_hash_t *hs, void *old_key,
                          void *new_key, struct hlist_node *hnode);
+/** @} */
 
 
 #define CFS_HASH_THETA_BITS  10
 
-/* Return integer component of theta */
+/** \return integer component of theta */
 static inline int __cfs_hash_theta_int(int theta)
 {
         return (theta >> CFS_HASH_THETA_BITS);
 }
 
-/* Return a fractional value between 0 and 999 */
+/** \return a fractional value between 0 and 999 */
 static inline int __cfs_hash_theta_frac(int theta)
 {
         return ((theta * 1000) >> CFS_HASH_THETA_BITS) -
@@ -364,11 +473,22 @@ static inline void __cfs_hash_set_theta(
         hs->hs_max_theta = max;
 }
 
-/* Generic debug formatting routines mainly for proc handler */
+/**
+ *\defgroup hash_debug Hash debug functions
+ *
+ * Generic debug formatting routines mainly for proc handler
+ * @{
+ */
 int cfs_hash_debug_header(char *str, int size);
 int cfs_hash_debug_str(cfs_hash_t *hs, char *str, int size);
+/** @} */
 
-/*
+/**
+ *\defgroup hash_algs Generic hash algorithms
+ * @{
+ */
+
+/**
  * Generic djb2 hash algorithm for character arrays.
  */
 static inline unsigned
@@ -384,7 +504,7 @@ cfs_hash_djb2_hash(void *key, size_t siz
         return (hash & mask);
 }
 
-/*
+/**
  * Generic u32 hash algorithm.
  */
 static inline unsigned
@@ -393,7 +513,7 @@ cfs_hash_u32_hash(__u32 key, unsigned ma
         return ((key * CFS_GOLDEN_RATIO_PRIME_32) & mask);
 }
 
-/*
+/**
  * Generic u64 hash algorithm.
  */
 static inline unsigned
@@ -401,6 +521,7 @@ cfs_hash_u64_hash(__u64 key, unsigned ma
 {
         return ((unsigned)(key * CFS_GOLDEN_RATIO_PRIME_64) & mask);
 }
+/** @} hash_algs */
 
 #define cfs_hash_for_each_bucket(hs, hsb, pos)   \
         for (pos = 0;                            \
@@ -408,5 +529,8 @@ cfs_hash_u64_hash(__u64 key, unsigned ma
              ({ hsb = hs->hs_buckets[i]; 1; });  \
              pos++)
 
+/** @} hash_table */
+/** @} libcfs */
+
 /* !__LIBCFS__HASH_H__ */
 #endif
Index: libcfs/include/libcfs/libcfs_ioctl.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_ioctl.h,v
retrieving revision 1.3
diff -u -p -r1.3 libcfs_ioctl.h
--- libcfs/include/libcfs/libcfs_ioctl.h	7 Aug 2008 14:48:45 -0000	1.3
+++ libcfs/include/libcfs/libcfs_ioctl.h	1 Jan 2010 20:03:32 -0000
@@ -43,40 +43,72 @@
 #ifndef __LIBCFS_IOCTL_H__
 #define __LIBCFS_IOCTL_H__
 
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_ioctl
+ * @{ */
 
+/** Libcfs magic for sanity checks of \a libcfs_ioctl_data */
 #define LIBCFS_IOCTL_VERSION 0x0001000a
 
+/**
+ * Format of ioctl data that user-space ioctl callers and in-kernel
+ * ioctl handlers must share.
+ *
+ * It's actually the head of ioctl data. The whole payload looks like this:
+ * - \a libcfs_ioctl_data
+ * - the first inline buffer of \a ioc_inllen1 bytes (must be zero-terminated)
+ * - the second inline buffer of \a ioc_inllen2 bytes (must be zero-terminated)
+ */
 struct libcfs_ioctl_data {
+        /** total length of payload including inline buffers */
         __u32 ioc_len;
+        /** LIBCFS_IOCTL_VERSION magic */
         __u32 ioc_version;
 
+        /** any 8-bytes value */
         __u64 ioc_nid;
+        /** any 8-bytes value */
         __u64 ioc_u64[1];
 
+        /** any 4-bytes value */
         __u32 ioc_flags;
+        /** any 4-bytes value */
         __u32 ioc_count;
+        /** any 4-bytes value */
         __u32 ioc_net;
+        /** any seven 4-bytes values */
         __u32 ioc_u32[7];
 
+        /** size of the first inline buffer */
         __u32 ioc_inllen1;
+        /** pointer to the first inline buffer */
         char *ioc_inlbuf1;
+        /** size of the second inline buffer */
         __u32 ioc_inllen2;
+        /** pointer to the second inline buffer */
         char *ioc_inlbuf2;
 
-        __u32 ioc_plen1; /* buffers in userspace */
+        /** size of the first user-space buffer */
+        __u32 ioc_plen1;
+        /** pointer to the first user-space buffer */
         char *ioc_pbuf1;
-        __u32 ioc_plen2; /* buffers in userspace */
+        /** size of the second user-space buffer */
+        __u32 ioc_plen2;
+        /** pointer to the second user-space buffer */
         char *ioc_pbuf2;
 
+        /** the first inline buffer may start here */
         char ioc_bulk[0];
 };
 
-
+/** duplicates the head of \a libcfs_ioctl_data */
 struct libcfs_ioctl_hdr {
         __u32 ioc_len;
         __u32 ioc_version;
 };
 
+/** format of ioctl data for ioctl(LIBCFS_IOC_DEBUG_MASK) */
 struct libcfs_debug_ioctl_data
 {
         struct libcfs_ioctl_hdr hdr;
@@ -84,6 +116,7 @@ struct libcfs_debug_ioctl_data
         unsigned int debug;
 };
 
+/** helper for initializing \a libcfs_ioctl_data in user-space */
 #define LIBCFS_IOC_INIT(data)                           \
 do {                                                    \
         memset(&data, 0, sizeof(data));                 \
@@ -93,11 +126,16 @@ do {                                    
 
 #ifdef __KERNEL__
 
+/** wrapper over ioctl handler that any in-kernel consumer of Libcfs
+ * should use to register some ioctl handler in Libcfs */
 struct libcfs_ioctl_handler {
+        /** list node to be element of global ioctl_list */
         struct list_head item;
+        /** consumer-defined ioctl handler */
         int (*handle_ioctl)(unsigned int cmd, struct libcfs_ioctl_data *data);
 };
 
+/** shortcut to declare and initialize \a libcfs_ioctl_handler in one line */
 #define DECLARE_IOCTL_HANDLER(ident, func)                      \
         struct libcfs_ioctl_handler ident = {                   \
                 /* .item = */ CFS_LIST_HEAD_INIT(ident.item),   \
@@ -226,4 +264,7 @@ extern int libcfs_ioctl_popdata(void *ar
 
 #endif 
 
+/** @} libcfs_ioctl */
+/** @} libcfs */
+
 #endif /* __LIBCFS_IOCTL_H__ */
Index: libcfs/include/libcfs/libcfs_kernelcomm.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_kernelcomm.h,v
retrieving revision 1.2
diff -u -p -r1.2 libcfs_kernelcomm.h
--- libcfs/include/libcfs/libcfs_kernelcomm.h	19 Sep 2009 18:36:17 -0000	1.2
+++ libcfs/include/libcfs/libcfs_kernelcomm.h	1 Jan 2010 20:03:32 -0000
@@ -51,23 +51,48 @@
 #error Do not #include this file directly. #include <libcfs/libcfs.h> instead
 #endif
 
-/* LNL message header.
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup libcfs_lnl Kernel <-> user-space communication
+ * Kernel <-> user-space communication routines.
+ *
+ * We'll use a shorthand term "lnl" (Lustre NetLink) for this interface name
+ * for all arches, even though an implementation may not use NetLink.
+ *
+ * The definitions below are used in the kernel and user-space.
+ * @{ */
+
+/**
+ * LNL message header.
  * All current and future LNL messages should use this header.
  * To avoid having to include Lustre headers from libcfs, define this here.
  */
 struct lnl_hdr {
+        /** magic value for sanity checks */
         __u16 lnl_magic;
-        __u8  lnl_transport;  /* Each new Lustre feature should use a different
-                                 transport */
+        /** Each new Lustre feature should use a different transport */
+        __u8  lnl_transport;  
+        /** block/non-block behavior */                        
         __u8  lnl_flags;
-        __u16 lnl_msgtype;    /* Message type or opcode, transport-specific */
+        /** Message type or opcode, transport-specific */
+        __u16 lnl_msgtype;
+        /** total length of LNL message including this header */
         __u16 lnl_msglen;
 } __attribute__((aligned(sizeof(__u64))));
 
-#define LNL_MAGIC  0x191C /*Lustre9etLinC */
-#define LNL_FL_BLOCK 0x01   /* Wait for send */
+/** Lustre9etLinC. Used for lnl_hdr::lnl_magic. */
+#define LNL_MAGIC  0x191C 
+
+/** Wait for send. Used for lnl_hdr::lnl_flags. */
+#define LNL_FL_BLOCK 0x01
 
-/* lnl_msgtype values are defined in each transport */
+/**
+ * Enumerates all known transports. See description of lnl_hdr::lnl_transport.
+ *
+ * lnl_msgtype values are defined in each transport
+ */
 enum lnl_transport_type {
         LNL_TRANSPORT_GENERIC   = 1,
         LNL_TRANSPORT_HSM       = 2,
@@ -84,10 +109,14 @@ enum lnl_generic_message_type {
  * groups need not be used if e.g. a PID is specified instead;
  * use group 0 to signify unicast.
  */
+/** broadcast group for HSM */
 #define LNL_GRP_HSM           0x02
+/** total number of supported broadcast groups */
 #define LNL_GRP_CNT              2
 
-
+/** \defgroup libcfs_kernel_lnl Kernel <-> user-space communication (kernel)
+ * Kernel side of kernel <-> user-space communication subsystem.
+ * @{ */
 #if defined(HAVE_NETLINK) && defined (__KERNEL__)
 extern int libcfs_klnl_start(int transport);
 extern int libcfs_klnl_stop(int transport, int group);
@@ -103,8 +132,9 @@ static inline int libcfs_klnl_msg_put(in
         return -ENOSYS;
 }
 #endif
+/** @} libcfs_kernel_lnl */
 
-/*
+/**
  * NetLink socket number, see include/linux/netlink.h
  * All LNL users share a single netlink socket.  This actually is NetLink
  * specific, but is not to be used outside of the Linux implementation
@@ -112,6 +142,8 @@ static inline int libcfs_klnl_msg_put(in
  */
 #define LNL_SOCKET 26
 
+/** @} libcfs_lnl */
+/** @} libcfs */
 
 #endif /* __LIBCFS_KERNELCOMM_H__ */
 
Index: libcfs/include/libcfs/libcfs_prim.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_prim.h,v
retrieving revision 1.7
diff -u -p -r1.7 libcfs_prim.h
--- libcfs/include/libcfs/libcfs_prim.h	1 Dec 2009 02:23:43 -0000	1.7
+++ libcfs/include/libcfs/libcfs_prim.h	1 Jan 2010 20:03:32 -0000
@@ -42,46 +42,190 @@
 #ifndef __LIBCFS_PRIM_H__
 #define __LIBCFS_PRIM_H__
 
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup libcfs_misc Misc.
+ * Miscellaneous Libcfs primitives
+ * @{
+ */
+
+/**
+ * On some platforms an explicit directive is required to make a
+ * symbol (function/global/etc) visible outside given kernel module.
+ */
 #ifndef CFS_EXPORT_SYMBOL
 # define CFS_EXPORT_SYMBOL(s)
 #endif
+/** @} libcfs_misc */
 
-/*
- * Schedule
+/**
+ * \defgroup libcfs_schedule Schedule
+ * Generic API for interaction with os scheduler
+ *
+ * Sometimes a task may decide to postpone further execution for a
+ * while:
+ * - when it has nothing to do right now and a pause needed
+ * - when it has accomplished some amount of work and wants to give
+ *   other task a chance to proceed
+ *
+ * API below is various flavors of such rescheduling.
+ * @{
+ */
+
+/**
+ * Pausing for \a timeout.
+ *
+ * Task state will be set to \a state before
+ * give up execution control if it's doable on given platform
  */
 void cfs_schedule_timeout(cfs_task_state_t state, int64_t timeout);
+
+/**
+ * Ask os scheduler to switch to some task sitting in run-queue.
+ *
+ * NOTE: if no other tasks present, it returns immediately.
+ */
 void cfs_schedule(void);
+
+/** pausing for \a ticks in uninterruptible way */
 void cfs_pause(cfs_duration_t ticks);
+
+/** returns non-zero if any other task presents in run-queue */
 int  cfs_need_resched(void);
+
+/** reschedule only if any other task presents in run-queue */
 void cfs_cond_resched(void);
+/** @} libcfs_schedule */
 
-/*
- * Wait Queues
+/** \addtogroup libcfs_synchronization
+ * @{ */
+
+/**
+ * \defgroup libcfs_waitqueue Wait Queues
+ * Generic API for wait queues
+ *
+ * The core concepts are 'wait-queue' as represented by \a cfs_waitq_t
+ * and 'wait-link' as represented by \a cfs_waitlink_t. Their definitions
+ * are platform-specific while API of operations on them is generic.
+ * Wait-queue is essentially the list of waiters where every waiter is
+ * described by wait-link. Typically, when several threads are
+ * contending for some resource, there is one global wait-queue
+ * accessible from each thread and initialized early (before any thread
+ * has a chance to access it). Then, one or more threads may create and
+ * initialize wait-link (specific for given thread), then associate it
+ * with wait-queue and go to sleep using cfs_waitq_wait or
+ * cfs_waitq_timedwait primitives. Then, any other thread may signal
+ * wait-queue to wake up one or more waiters (i.e threads that
+ * associated their wait-links with wait-queue).
+ *
+ * The caller must use some lock in conjunction with wait-queue
+ * primitives to avoid 'lost wakeup' problem. Typical usage is to
+ * check for resource and add wait-link to wait-queue inside critical
+ * section protected by that lock. Then release the lock and
+ * go to sleep (e.g. with cfs_waitq_wait). When signalled,
+ * re-acquire the lock and check for resource again. Of course,
+ * signalling thread also must use that lock for updating
+ * resource before signalling.
+ *
+ * NOTE: lock strategy for wait-queue primitives taking condition as argument
+ * is different. See documentation of cfs_waitq_wait_event_timeout for details.
+ *
+ * NOTE: It's responsibility of caller to switch task state to 'interruptible'
+ * or 'uninterruptible' before calling  cfs_waitq_timedwait and cfs_waitq_wait
+ * and turn it back to 'running' afterwards.
+ *
+ * Consumer of wait-queue primitives may choose between exclusive and
+ * non-exclusive semantics. 'Exclusive' means intention to wake up only
+ * some number of waiters (usually one), 'non-exclusive' - to wake up
+ * all. To gain exclusive semantic the consumer must use both
+ * cfs_waitq_add_exclusive and cfs_waitq_signal (or cfs_waitq_signal_nr
+ * with \a nr > 0). Otherwise all waiters will be woken up.
+ *
+ * \pre wait-queue must be initialized with cfs_waitq_init prior any
+ * operation on it.
+ * \pre wait-link must be initialized with cfs_waitlink_init prior
+ * any operation on it.
+ * @{
  */
+
+/** initialize wait-queue */
 void cfs_waitq_init(cfs_waitq_t *waitq);
+/** initialize wait-link */
 void cfs_waitlink_init(cfs_waitlink_t *link);
+/** associate wait-link with given wait-queue */
 void cfs_waitq_add(cfs_waitq_t *waitq, cfs_waitlink_t *link);
+/** associate wait-link with given wait-queue enabling exclusive semantic */
 void cfs_waitq_add_exclusive(cfs_waitq_t *waitq,
                              cfs_waitlink_t *link);
+/** dissociate wait-link from wait-queue */
 void cfs_waitq_del(cfs_waitq_t *waitq, cfs_waitlink_t *link);
+/** \return 1 if wait-queue has waiters, 0 if otherwise */
 int  cfs_waitq_active(cfs_waitq_t *waitq);
+/** wake up one waiter if exclusive, wake up all otherwise */
 void cfs_waitq_signal(cfs_waitq_t *waitq);
+/** wake up \a nr waiters if exclusive, wake up all otherwise */
 void cfs_waitq_signal_nr(cfs_waitq_t *waitq, int nr);
+/** wake up all waiters */
 void cfs_waitq_broadcast(cfs_waitq_t *waitq);
+/** wait till signalled */
 void cfs_waitq_wait(cfs_waitlink_t *link, cfs_task_state_t state);
+/**
+ * wait till signalled but no longer than \a timeout
+ * \return 0 if the whole \a timeout lapsed, number of machine ticks
+ * remaining before timeout otherwise
+ */
 int64_t cfs_waitq_timedwait(cfs_waitlink_t *link, cfs_task_state_t state, 
 			    int64_t timeout);
+/** @} libcfs_waitqueue */
+/** @} libcfs_synchronization */
+
+/**
+ * \defgroup libcfs_timer Timer
+ * Generic API for manipulating timers
+ *
+ * cfs_timer_t is platform-specific structure carrying info:
+ * - what function to be called
+ * - with what argument
+ * - when
+ *
+ * Main APIs includes initialization/finalization and
+ * activation/deactivation. After activation, the caller assumes
+ * that the function (registered with cfs_timer_init) will be called
+ * when \a deadline comes.
+ *
+ * \pre Timer must be initialized with cfs_timer_init prior calling
+ *      any other timer API.
+ *
+ * NOTE: cfs_init_timer can also be used for initialization, but in such
+ * case \a func and \a private_data parts of timer structure stay
+ * uninitialized. That makes cfs_init_timer quite useless, so one
+ * should prefer to use cfs_timer_init instead.
+ * @{
+ */
 
-/*
- * Timer
+/**
+ * Callback to be called when timer is fired. Will be called with
+ * opaque pointer (private data) passed as last argument to cfs_timer_init.
  */
 typedef  void (cfs_timer_func_t)(ulong_ptr_t);
 
+/** deprecated */
 void cfs_init_timer(cfs_timer_t *t);
+/** initialize timer */
 void cfs_timer_init(cfs_timer_t *t, cfs_timer_func_t *func, void *arg);
+/** finalize timer */
 void cfs_timer_done(cfs_timer_t *t);
+/** activate timer */
 void cfs_timer_arm(cfs_timer_t *t, cfs_time_t deadline);
+/** deactivate timer */
 void cfs_timer_disarm(cfs_timer_t *t);
+/** ask if timer is activated */
 int  cfs_timer_is_armed(cfs_timer_t *t);
+/** \return expiration time of given timer */
 cfs_time_t cfs_timer_deadline(cfs_timer_t *t);
+/** @} libcfs_timer */
+
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/libcfs_private.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_private.h,v
retrieving revision 1.14
diff -u -p -r1.14 libcfs_private.h
--- libcfs/include/libcfs/libcfs_private.h	1 Dec 2009 02:23:43 -0000	1.14
+++ libcfs/include/libcfs/libcfs_private.h	1 Jan 2010 20:03:32 -0000
@@ -45,15 +45,79 @@
 /* XXX this layering violation is for nidstrings */
 #include <lnet/types.h>
 
+/** \addtogroup libcfs
+ * @{ */
+
 #ifndef DEBUG_SUBSYSTEM
 # define DEBUG_SUBSYSTEM S_UNDEFINED
 #endif
 
+/** \addtogroup libcfs_debug
+ * @{ */
+
+/**
+ * \defgroup libcfs_assert Assertions
+ * Assertions macros.
+ *
+ * Asserting is a programming technique of adding special fragments of
+ * code that check whether some condition is true, and, if it's not,
+ * triggering debug actions like error prints or dumping logs and
+ * stopping execution of the program.
+ *
+ * For user-space, Libcfs implements all kinds of assertion macros via
+ * assert(3) that just prints an error message to stderr and
+ * terminates the program by calling abort(3) if condition is false.
+ *
+ * For in-kernel users, assertion macros (when condition is false)
+ * generate non-maskable debug print that goes both on console and to
+ * internal log buffers and trigger so-called "LBUG". In interrupt
+ * context, LBUG is handled as panic the system. In other cases, it
+ * results in dumping all internal logs gathered by that moment and
+ * either suspending execution thread (where this assertion failed)
+ * forever or system panic depending on the value of global variable
+ * \a libcfs_panic_on_lbug (it's 1 by default that means "panic always").
+ *
+ * The consumer may also use LBUG directly in parts of code to stress
+ * the fact that execution control should never reach this place in
+ * "normal" situation (i.e. if the whole code is working as designed).
+ * In such a case, LBUG is equivalent to an assert with false condition.
+ *
+ * Libcfs provides three flavors of assertions suitable for various
+ * purposes:
+ *
+ * 1. General light-weight assertions to be used when checking
+ * conditions doesn't involve significant impact on performance:
+ * - LASSERT - just "assertion" as explained above
+ * - LASSERTF - the same as previous but also prints specified error
+ * message in logs
+ * - KLASSERT - the same as LASSERT but never checked in user-space;
+ * useful in parts of code to be compiled both for kernel and
+ * user-space when checking condition is meaningful only for in-kernel
+ * use.
+ *
+ * 2. Assertions for expensive checks that one doesn't want to be
+ * enabled all the time:
+ * - LINVRNT - the same as LASSERT but enabled only for builds with
+ * INVARIANT_CHECK defined.
+ *
+ * 3. Compile-time assertion:
+ * - CLASSERT - to be used when condition can be evaluated while compiling
+ */
+
+/** @} libcfs_debug */
+
 #ifdef __KERNEL__
 
 #ifdef LIBCFS_DEBUG
 
-/*
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_assert
+ * @{
+ * \defgroup libcfs_kernel_assert Assertions (kernel-space)
+ * @{ */
+
+/**
  * When this is on, LASSERT macro includes check for assignment used instead
  * of equality check, but doesn't have unlikely(). Turn this on from time to
  * time to make test-builds. This shouldn't be on for production release.
@@ -61,7 +125,7 @@
 #define LASSERT_CHECKED (0)
 
 #if LASSERT_CHECKED
-/*
+/**
  * Assertion.
  *
  * Strange construction with empty "then" clause is used to trigger compiler
@@ -138,10 +202,17 @@ void lbug_with_loc(const char *file, con
 
 #define LBUG() lbug_with_loc(__FILE__, __FUNCTION__, __LINE__)
 
+/** @} libcfs_kernel_assert */
+/** @} libcfs_assert */
+/** @} libcfs_debug */
+
+/** \addtogroup libcfs_mem
+ * @{
+ * \addtogroup libcfs_kernel_mem
+ * @{ */
+
 extern atomic_t libcfs_kmemory;
-/*
- * Memory
- */
+
 #ifdef LIBCFS_DEBUG
 
 # define libcfs_kmem_inc(ptr, size)             \
@@ -158,8 +229,11 @@ do {                                    
 # define libcfs_kmem_dec(ptr, size) do {} while (0)
 #endif /* LIBCFS_DEBUG */
 
+/** limit to switch to vm allocations */
 #define LIBCFS_VMALLOC_SIZE        16384
 
+/** a macro for internal libcfs use that allows to specify allocation
+ * type directly (by \a mask argument) */
 #define LIBCFS_ALLOC_GFP(ptr, size, mask)                                 \
 do {                                                                      \
         LASSERT(!in_interrupt() ||                                        \
@@ -206,6 +280,13 @@ do {                                    
 } while (0)
 
 /******************************************************************************/
+/** @} libcfs_kernel_mem */
+/** @} libcfs_mem */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \defgroup libcfs_kernel_misc (platform-neutral, kernel-space)
+ * @{ */
 
 /* htonl hack - either this, or compile with -O2. Stupid byteorder/generic.h */
 #if defined(__GNUC__) && (__GNUC__ >= 2) && !defined(__OPTIMIZE__)
@@ -219,6 +300,14 @@ do {                                    
 #define ntohs(x) ___ntohs(x)
 #endif
 
+/** @} libcfs_kernel_misc */
+/** @} libcfs_misc */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \defgroup libcfs_kernel_debug (platform-neutral, kernel-space)
+ * @{ */
+
 void libcfs_debug_dumpstack(cfs_task_t *tsk);
 void libcfs_run_upcall(char **argv);
 void libcfs_run_lbug_upcall(const char * file, const char *fn, const int line);
@@ -230,7 +319,18 @@ int libcfs_debug_mark_buffer(const char 
 
 void libcfs_debug_set_level(unsigned int debug_level);
 
+/** @} libcfs_kernel_debug */
+/** @} libcfs_debug */
+
 #else  /* !__KERNEL__ */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_assert
+ * @{
+ * \defgroup libcfs_user_assert Assertions (user-space)
+ * @{ */
+
 # ifdef LIBCFS_DEBUG
 #  undef NDEBUG
 #  include <assert.h>
@@ -254,7 +354,21 @@ do {                                    
 #  define LINVRNT(exp) ((void)sizeof!!(exp))
 # endif /* LIBCFS_DEBUG */
 # define KLASSERT(e) ((void)0)
+/** @} libcfs_user_assert */
+/** @} libcfs_assert */
+
+/** \addtogroup libcfs_debug_print
+ * @{ */
 # define printk printf
+/** @} libcfs_debug_print */
+
+/** @} libcfs_debug */
+
+
+/** \addtogroup libcfs_mem
+ * @{
+ * \addtogroup libcfs_user_mem
+ * @{ */
 # ifdef CRAY_XT3                                /* buggy calloc! */
 #  define LIBCFS_ALLOC(ptr, size)               \
    do {                                         \
@@ -265,6 +379,13 @@ do {                                    
 #  define LIBCFS_ALLOC(ptr, size) do { (ptr) = calloc(1,size); } while (0)
 # endif
 # define LIBCFS_FREE(a, b) do { free(a); } while (0)
+/** @} libcfs_user_mem */
+/** @} libcfs_mem */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_user_debug
+ * @{ */
 
 void libcfs_debug_dumplog(void);
 int libcfs_debug_init(unsigned long bufsize);
@@ -272,6 +393,14 @@ int libcfs_debug_cleanup(void);
 
 #define libcfs_debug_dumpstack(tsk)     ((void)0)
 
+/** @} libcfs_user_debug */
+/** @} libcfs_debug */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_userspace_misc
+ * @{ */
+
 /*
  * Generic compiler-dependent macros required for kernel
  * build go below this comment. Actual compiler/compiler version
@@ -284,11 +413,23 @@ int libcfs_debug_cleanup(void);
 #define likely(x)	(!!(x))
 #define unlikely(x)	(!!(x))
 #endif
+
+/** @} libcfs_userspace_misc */
+/** @} libcfs_misc */
+
 /* !__KERNEL__ */
 #endif
 
+/** \addtogroup libcfs_mem
+ * @{ */
 #define CFS_ALLOC_PTR(ptr)      LIBCFS_ALLOC(ptr, sizeof (*(ptr)));
 #define CFS_FREE_PTR(ptr)       LIBCFS_FREE(ptr, sizeof (*(ptr)));
+/** @} libcfs_mem */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_assert
+ * @{ */
 
 /** Compile-time assertion. 
 
@@ -305,43 +446,15 @@ int libcfs_debug_cleanup(void);
  *
  */
 #define CLASSERT(cond) do {switch(42) {case (cond): case 0: break;}} while (0)
+/** @} libcfs_assert */
+/** @} libcfs_debug */
 
-/* support decl needed both by kernel and liblustre */
-int         libcfs_isknown_lnd(int type);
-char       *libcfs_lnd2modname(int type);
-char       *libcfs_lnd2str(int type);
-int         libcfs_str2lnd(const char *str);
-char       *libcfs_net2str(__u32 net);
-char       *libcfs_nid2str(lnet_nid_t nid);
-__u32       libcfs_str2net(const char *str);
-lnet_nid_t  libcfs_str2nid(const char *str);
-int         libcfs_str2anynid(lnet_nid_t *nid, const char *str);
-char       *libcfs_id2str(lnet_process_id_t id);
-int         cfs_iswhite(char c);
-void        cfs_free_nidlist(struct list_head *list);
-int         cfs_parse_nidlist(char *str, int len, struct list_head *list);
-int         cfs_match_nid(lnet_nid_t nid, struct list_head *list);
-
-/* how an LNET NID encodes net:address */
-#define LNET_NIDADDR(nid)      ((__u32)((nid) & 0xffffffff))
-#define LNET_NIDNET(nid)       ((__u32)(((nid) >> 32)) & 0xffffffff)
-#define LNET_MKNID(net,addr)   ((((__u64)(net))<<32)|((__u64)(addr)))
-/* how net encodes type:number */
-#define LNET_NETNUM(net)       ((net) & 0xffff)
-#define LNET_NETTYP(net)       (((net) >> 16) & 0xffff)
-#define LNET_MKNET(typ,num)    ((((__u32)(typ))<<16)|((__u32)(num)))
-
-/* max value for numeric network address */
-#define MAX_NUMERIC_VALUE 0xffffffff
-
-/* implication */
-#define ergo(a, b) (!(a) || (b))
-/* logical equivalence */
-#define equi(a, b) (!!(a) == !!(b))
-
+/** \addtogroup libcfs_time
+ * @{ */
 #ifndef CURRENT_TIME
 # define CURRENT_TIME time(0)
 #endif
+/** @} libcfs_time */
 
 /* --------------------------------------------------------------------
  * Light-weight trace
@@ -349,11 +462,20 @@ int         cfs_match_nid(lnet_nid_t nid
  * All stuff about lwt are put in arch/kp30.h
  * -------------------------------------------------------------------- */
 
+/** \addtogroup libcfs_ioctl
+ * @{ */
+/** user-state info */
 struct libcfs_device_userstate
 {
+        /** # memhog pages allocated */
         int           ldu_memhog_pages;
+        /** pointer to page containing layout of memhog pages */
         cfs_page_t   *ldu_memhog_root_page;
 };
+/** @} libcfs_ioctl */
+
+/** \addtogroup libcfs_misc
+ * @{ */
 
 /* what used to be in portals_lib.h */
 #ifndef MIN
@@ -440,4 +562,39 @@ enum {
         MXLND     = 12,
 };
 
+/* support decl needed both by kernel and liblustre */
+int         libcfs_isknown_lnd(int type);
+char       *libcfs_lnd2modname(int type);
+char       *libcfs_lnd2str(int type);
+int         libcfs_str2lnd(const char *str);
+char       *libcfs_net2str(__u32 net);
+char       *libcfs_nid2str(lnet_nid_t nid);
+__u32       libcfs_str2net(const char *str);
+lnet_nid_t  libcfs_str2nid(const char *str);
+int         libcfs_str2anynid(lnet_nid_t *nid, const char *str);
+char       *libcfs_id2str(lnet_process_id_t id);
+int         cfs_iswhite(char c);
+void        cfs_free_nidlist(struct list_head *list);
+int         cfs_parse_nidlist(char *str, int len, struct list_head *list);
+int         cfs_match_nid(lnet_nid_t nid, struct list_head *list);
+
+/* how an LNET NID encodes net:address */
+#define LNET_NIDADDR(nid)      ((__u32)((nid) & 0xffffffff))
+#define LNET_NIDNET(nid)       ((__u32)(((nid) >> 32)) & 0xffffffff)
+#define LNET_MKNID(net,addr)   ((((__u64)(net))<<32)|((__u64)(addr)))
+/* how net encodes type:number */
+#define LNET_NETNUM(net)       ((net) & 0xffff)
+#define LNET_NETTYP(net)       (((net) >> 16) & 0xffff)
+#define LNET_MKNET(typ,num)    ((((__u32)(typ))<<16)|((__u32)(num)))
+
+/* max value for numeric network address */
+#define MAX_NUMERIC_VALUE 0xffffffff
+
+/* implication */
+#define ergo(a, b) (!(a) || (b))
+/* logical equivalence */
+#define equi(a, b) (!!(a) == !!(b))
+
+/** @} libcfs_misc */
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/libcfs_string.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_string.h,v
retrieving revision 1.2
diff -u -p -r1.2 libcfs_string.h
--- libcfs/include/libcfs/libcfs_string.h	13 Mar 2009 08:17:27 -0000	1.2
+++ libcfs/include/libcfs/libcfs_string.h	1 Jan 2010 20:03:32 -0000
@@ -43,7 +43,12 @@
 #ifndef __LIBCFS_STRING_H__
 #define __LIBCFS_STRING_H__
 
-/* libcfs_string.c */
+/** \addtogroup libcfs
+ * @{
+ * \defgroup libcfs_string String
+ * Generic API for manipulating strings.
+ * @{ */
+ 
 /* Convert a text string to a bitmask */
 int libcfs_str2mask(const char *str, const char *(*bit2str)(int bit),
                     int *oldmask, int minmask, int allmask);
@@ -52,4 +57,7 @@ int libcfs_str2mask(const char *str, con
  * Must free with cfs_free().
  */
 char *libcfs_strdup(const char *str, u_int32_t flags);
+
+/** @} libcfs_string */
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/libcfs_time.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/libcfs_time.h,v
retrieving revision 1.4
diff -u -p -r1.4 libcfs_time.h
--- libcfs/include/libcfs/libcfs_time.h	7 Oct 2008 04:14:06 -0000	1.4
+++ libcfs/include/libcfs/libcfs_time.h	1 Jan 2010 20:03:32 -0000
@@ -41,8 +41,40 @@
 
 #ifndef __LIBCFS_TIME_H__
 #define __LIBCFS_TIME_H__
-/*
- * generic time manipulation functions.
+
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup libcfs_time Time
+ * Generic time manipulation functions.
+ *
+ * There are three main concepts related to libcfs time manipulation:
+ * - A point in time represented as number of machine ticks lapsed
+ * since certain moment (usually since last reboot). Libcfs uses
+ * \a cfs_time_t for this purpose.
+ * - Wall-clock time. Libcfs uses \a time_t or \a timeval for this purpose.
+ * In \a time_t case, the unit of measure is the second. Not to be
+ * confused with \a cfs_time_t measured in ticks.
+ * - Duration. This is the difference between two certain points in
+ * time. Libcfs uses \a cfs_duration_t when it's measured in ticks. It can
+ * be converted to seconds and vice-versa using
+ * \a cfs_duration_sec/cfs_time_seconds APIs. Libcfs doesn't provide any
+ * special API for handling \a time_t times (i.e. measured in seconds)
+ * because the caller may use common arithmetic ('-', '<', etc) to
+ * calculate duration or compare times in this case.
+ *
+ * The simplest way for understanding libcfs time API and using it
+ * correctly is to pay special attention to types of arguments and
+ * return values (not relying on compiler warnings). For example, the
+ * first argument of \a cfs_time_add is of \a cfs_time_t type. So,
+ * \a cfs_time_current can be used as source for this argument, but
+ * \a cfs_time_current_sec cannot.
+ *
+ * NOTE: If the caller needs monotonic time, the only option is to
+ * use \a cfs_time_current. Other APIs (like \a cfs_time_current_sec)
+ * won't do.
+ * @{
  */
 
 static inline cfs_time_t cfs_time_add(cfs_time_t t, cfs_duration_t d)
@@ -94,6 +126,9 @@ static inline long cfs_timeval_sub(struc
         return r;
 }
 
+/**
+ * Obsoleted; Nobody uses it.
+ */
 static inline void cfs_slow_warning(cfs_time_t now, int seconds, char *msg)
 {
         if (cfs_time_after(cfs_time_current(),
@@ -102,6 +137,9 @@ static inline void cfs_slow_warning(cfs_
                        cfs_duration_sec(cfs_time_sub(cfs_time_current(),now)));
 }
 
+/**
+ * Obsoleted; Nobody uses it.
+ */
 #define CFS_RATELIMIT(seconds)                                  \
 ({                                                              \
         /*                                                      \
@@ -119,7 +157,7 @@ static inline void cfs_slow_warning(cfs_
         result;                                                 \
 })
 
-/*
+/**
  * helper function similar to do_gettimeofday() of Linux kernel
  */
 static inline void cfs_fs_timeval(struct timeval *tv)
@@ -130,7 +168,7 @@ static inline void cfs_fs_timeval(struct
         cfs_fs_time_usec(&time, tv);
 }
 
-/*
+/**
  * return valid time-out based on user supplied one. Currently we only check
  * that time-out is not shorted than allowed.
  */
@@ -141,4 +179,7 @@ static inline cfs_duration_t cfs_timeout
         return timeout;
 }
 
+/** @} libcfs_time */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/list.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/list.h,v
retrieving revision 1.7
diff -u -p -r1.7 list.h
--- libcfs/include/libcfs/list.h	26 Oct 2009 08:08:19 -0000	1.7
+++ libcfs/include/libcfs/list.h	1 Jan 2010 20:03:32 -0000
@@ -1,46 +1,99 @@
 #ifndef __LIBCFS_LIST_H__
 #define __LIBCFS_LIST_H__
 
+/** \addtogroup libcfs
+ * @{ */
+
+/** \defgroup lists Lists
+ * @{ */
+
+/**
+ * \defgroup list List
+ * Simple doubly linked list implementation.
+ *
+ * Some of the internal functions ("__xxx") are useful when
+ * manipulating whole lists rather than single entries, as
+ * sometimes we already know the next/prev entries and we can
+ * generate better code by using them directly rather than
+ * using the generic single-entry routines.
+ *
+ * \defgroup hlist Hash List
+ * Double linked lists with a single pointer list head.
+ * Mostly useful for hash tables where the two pointer list head is too
+ * wasteful.  You lose the ability to access the tail in O(1).
+ */
+
 #if defined (__linux__) && defined(__KERNEL__)
 
 #include <linux/list.h>
 
+/** \addtogroup list
+ * @{ */
 #define CFS_LIST_HEAD_INIT(n)		LIST_HEAD_INIT(n)
 #define CFS_LIST_HEAD(n)		LIST_HEAD(n)
 #define CFS_INIT_LIST_HEAD(p)		INIT_LIST_HEAD(p)
+/* @} */
 
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+/** \addtogroup hlist
+ * @{ */
 #define CFS_HLIST_HEAD_INIT		HLIST_HEAD_INIT
 #define CFS_HLIST_HEAD(n)		HLIST_HEAD(n)
 #define CFS_INIT_HLIST_HEAD(p)		INIT_HLIST_HEAD(p)
 #define CFS_INIT_HLIST_NODE(p)		INIT_HLIST_NODE(p)
+/* @} */
 #endif
 
 #else /* !defined (__linux__) || !defined(__KERNEL__) */
 
-/*
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
+/**
+ * Prefetch the value of \a a to have it in CPU-cache in advance.
+ * Used in list manipulation primitives to increase performance.
  */
-
 #define prefetch(a) ((void)a)
 
+/** \addtogroup list
+ * @{ */
+
+/**
+ * There are two different use-cases for this structure:
+ *
+ * 1) An anchor keeping pointers necessary to iterate through the
+ * list forward starting from the first element and backward starting
+ * from the last.
+ * When used in this way, list_head should be initialized before use.
+ * The list is empty if and only if both \a next and \a prev point
+ * to the given list_head.
+ *
+ * 2) A storage (embedded in a structure describing list element) for
+ * pointers necessary for list manipulations like add, del, etc.
+ * No need of initialization in such use-case.
+ */
 struct list_head {
-	struct list_head *next, *prev;
+        /**
+         * points to:
+         *  - the first element of the list in the 1st use-case
+         *  - the next element of the list in the 2nd use-case
+         */
+	struct list_head *next;
+        /**
+         * points to:
+         *  - the last element of the list in the 1st use-case
+         *  - the previous element of the list in the 2nd use-case
+         */
+        struct list_head *prev;
 };
 
 typedef struct list_head list_t;
 
+/** Static initializer for list_head representing empty list */
 #define CFS_LIST_HEAD_INIT(name) { &(name), &(name) }
 
+/** Shortcut to declare and initialize list_head in one statement */
 #define CFS_LIST_HEAD(name) \
 	struct list_head name = CFS_LIST_HEAD_INIT(name)
 
+/** Initialize list_head at run time as representing empty list */
 #define CFS_INIT_LIST_HEAD(ptr) do { \
 	(ptr)->next = (ptr); (ptr)->prev = (ptr); \
 } while (0)
@@ -248,26 +301,37 @@ static inline void list_splice_init(stru
 #define list_for_each_safe(pos, n, head) \
 	for (pos = (head)->next, n = pos->next; pos != (head); \
 		pos = n, n = pos->next)
+/* @} list */
+
+/** \addtogroup hlist
+ * @{ */
 
 /**
- * \defgroup hlist Hash List
- * Double linked lists with a single pointer list head.
- * Mostly useful for hash tables where the two pointer list head is too
- * wasteful.  You lose the ability to access the tail in O(1).
- * @{
+ * An anchor keeping pointer to the first (\a first) element as start
+ * point for iteration through the list. hlist_head should be
+ * initialized before use.
+ * The list is empty if and only if \a first is equal to NULL.
  */
-
 struct hlist_head {
+        /** points to the first element of the hlist */
 	struct hlist_node *first;
 };
 
+/**
+ * A storage (embedded in a structure describing hlist element) for
+ * pointers necessary for list manipulations like add, del, etc.
+ * Should be initialized with CFS_INIT_HLIST_NODE to make hlist_unhashed
+ * working correctly. An element is regarded as 'unhashed' (i.e. not
+ * part of any hlist) if and only if \a pprev is equal to NULL.
+*/
 struct hlist_node {
-	struct hlist_node *next, **pprev;
+        /** points to the next element of the hlist */
+	struct hlist_node  *next;
+        /** points to the memory where \a next of previous element is stored */
+        struct hlist_node **pprev;
 };
 
-/* @} */
-
-/*
+/**
  * "NULL" might not be defined at this point
  */
 #ifdef NULL
@@ -276,14 +340,16 @@ struct hlist_node {
 #define NULL_P ((void *)0)
 #endif
 
-/**
- * \addtogroup hlist
- * @{
- */
-
+/** Static initializer for hlist_head representing empty hlist */
 #define CFS_HLIST_HEAD_INIT { NULL_P }
+
+/** Shortcut to declare and initialize hlist_head in one statement */
 #define CFS_HLIST_HEAD(name) struct hlist_head name = { NULL_P }
+
+/** Initialize hlist_head at run time as representing empty hlist */
 #define CFS_INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL_P)
+
+/** Initialize hlist_node at run time as representing 'unhashed' element */
 #define CFS_INIT_HLIST_NODE(ptr) ((ptr)->next = NULL_P, (ptr)->pprev = NULL_P)
 
 #define HLIST_HEAD_INIT		CFS_HLIST_HEAD_INIT
@@ -296,11 +362,13 @@ static inline int hlist_unhashed(const s
 	return !h->pprev;
 }
 
+/** the same as list_empty but for hlist */
 static inline int hlist_empty(const struct hlist_head *h)
 {
 	return !h->first;
 }
 
+/** the same as __list_del but for hlist */
 static inline void __hlist_del(struct hlist_node *n)
 {
 	struct hlist_node *next = n->next;
@@ -310,11 +378,17 @@ static inline void __hlist_del(struct hl
 		next->pprev = pprev;
 }
 
+/**
+ * the same as list_del but for hlist.
+ * NOTE: n->pprev is not zeroed, so hlist_unhashed won't work
+ * as expected afterwards. Use hlist_del_init if it's needed.
+ */
 static inline void hlist_del(struct hlist_node *n)
 {
 	__hlist_del(n);
 }
 
+/** the same as list_del_init but for hlist */
 static inline void hlist_del_init(struct hlist_node *n)
 {
 	if (n->pprev)  {
@@ -323,6 +397,7 @@ static inline void hlist_del_init(struct
 	}
 }
 
+/** the same as list_add but for hlist */
 static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
 {
 	struct hlist_node *first = h->first;
@@ -333,7 +408,13 @@ static inline void hlist_add_head(struct
 	n->pprev = &h->first;
 }
 
-/* next must be != NULL */
+/**
+ * Insert an entry immediately before entry specified by \a next.
+ * \param n new entry to be inserted
+ * \param next existing entry before which \a n to be inserted
+ *
+ * \a next must be != NULL
+ */
 static inline void hlist_add_before(struct hlist_node *n,
 					struct hlist_node *next)
 {
@@ -343,6 +424,13 @@ static inline void hlist_add_before(stru
 	*(n->pprev) = n;
 }
 
+/**
+ * Insert an entry immediately after entry specified by \a next.
+ * \param n new entry to be inserted
+ * \param next existing entry after which \a n to be inserted
+ *
+ * \a next must be != NULL
+ */
 static inline void hlist_add_after(struct hlist_node *n,
 					struct hlist_node *next)
 {
@@ -354,12 +442,15 @@ static inline void hlist_add_after(struc
 		next->next->pprev  = &next->next;
 }
 
+/** the same as list_entry but for hlist */
 #define hlist_entry(ptr, type, member) container_of(ptr,type,member)
 
+/** the same as list_for_each but for hlist */
 #define hlist_for_each(pos, head) \
 	for (pos = (head)->first; pos && (prefetch(pos->next), 1); \
 	     pos = pos->next)
 
+/** the same as list_for_each_safe but for hlist */
 #define hlist_for_each_safe(pos, n, head) \
 	for (pos = (head)->first; pos && (n = pos->next, 1); \
 	     pos = n)
@@ -414,10 +505,13 @@ static inline void hlist_add_after(struc
 		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
 	     pos = n)
 
-/* @} */
+/* @} hlist */
 
 #endif /* __linux__ && __KERNEL__ */
 
+/** \addtogroup list
+ * @{ */
+
 #ifndef list_for_each_prev
 /**
  * Iterate over a list in reverse order
@@ -446,6 +540,7 @@ static inline void hlist_add_after(struc
 #endif /* list_for_each_entry */
 
 #ifndef list_for_each_entry_rcu
+/** The same as ::list_for_each_entry but for rcu-protected lists */
 #define list_for_each_entry_rcu(pos, head, member) \
 	list_for_each_entry(pos, head, member)
 #endif
@@ -518,6 +613,10 @@ static inline void hlist_add_after(struc
         for (n = list_entry(pos->member.next, type, member);            \
              &pos->member != (head);                                    \
              pos = n, n = list_entry(n->member.next, type, member))
+/* @} list */
+
+/** \addtogroup hlist
+ * @{ */
 #define cfs_hlist_for_each_entry_typed(tpos, pos, head, type, member)   \
 	for (pos = (head)->first;                                       \
 	     pos && (prefetch(pos->next), 1) &&                         \
@@ -529,5 +628,8 @@ static inline void hlist_add_after(struc
 	     pos && (n = pos->next, 1) && 				\
 		(tpos = hlist_entry(pos, type, member), 1);             \
 	     pos = n)
+/* @} hlist  */
+/* @} lists */
+/* @} libcfs */
 
 #endif /* __LIBCFS_LUSTRE_LIST_H__ */
Index: libcfs/include/libcfs/lltrace.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/lltrace.h,v
retrieving revision 1.3
diff -u -p -r1.3 lltrace.h
--- libcfs/include/libcfs/lltrace.h	7 Aug 2008 14:48:45 -0000	1.3
+++ libcfs/include/libcfs/lltrace.h	1 Jan 2010 20:03:32 -0000
@@ -52,6 +52,26 @@
 #error Unsupported Operating System
 #endif
 
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup libcfs_lltrace lltrace
+ * Thin layer between jt_dbg_* interface provided by
+ * lnet/utils/debug.c and lustre/tests/mkdirdeep
+ *
+ * lltrace hides peculiarities of jt_dbg_* interface allowing
+ * some test program (lustre/tests/mkdirdeep) to set in-kernel libcfs
+ * debug tunables (subsystem_debug and debug) and dump lustre log in
+ * specified file in easy way.
+ *
+ * It also provides function \a ltrace_add_processnames that adds
+ * output of "ps -e" system command to the tail of dumped logs in the
+ * format alike to the rest of logs.
+ *
+ * @{
+ */
+
 static inline int ltrace_write_file(char* fname)
 {
         char* argv[3];
@@ -199,4 +219,7 @@ static inline void ltrace_add_processnam
         system(cmdbuf);
 }
 
+/** @} libcfs_lltrace */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/user-bitops.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-bitops.h,v
retrieving revision 1.6
diff -u -p -r1.6 user-bitops.h
--- libcfs/include/libcfs/user-bitops.h	8 Sep 2009 08:09:26 -0000	1.6
+++ libcfs/include/libcfs/user-bitops.h	1 Jan 2010 20:03:32 -0000
@@ -41,7 +41,25 @@
 #ifndef __LIBCFS_USER_BITOPS_H__
 #define __LIBCFS_USER_BITOPS_H__
 
-/* test if bit nr is set in bitmap addr; returns previous value of bit nr */
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_bitops
+ * @{ */
+
+/**
+ * \defgroup libcfs_userspace_bitops Bitmask (user-space)
+ * Bitmask operations (user-space).
+ *
+ * User-space implementation of bitmask primitives is NOT atomic. For
+ * example, two separate threads can easily race on test_and_set_bit,
+ * both returning zero if given bit was not set initially. It's not
+ * tidy but acceptable because the only consumer is single-threaded
+ * liblustre.
+ * @{
+ */
+
+/** set bit \a nr in bitmask \a addr
+ * \return previous value of bit \a nr */
 static __inline__ int test_and_set_bit(int nr, unsigned long *addr)
 {
         unsigned long mask;
@@ -55,7 +73,8 @@ static __inline__ int test_and_set_bit(i
 
 #define set_bit(n, a) test_and_set_bit(n, a)
 
-/* clear bit nr in bitmap addr; returns previous value of bit nr*/
+/** clear bit \a nr in bitmask \a addr
+ * \return previous value of bit \a nr */
 static __inline__ int test_and_clear_bit(int nr, unsigned long *addr)
 {
         unsigned long mask;
@@ -69,13 +88,24 @@ static __inline__ int test_and_clear_bit
 
 #define clear_bit(n, a) test_and_clear_bit(n, a)
 
+/** test bit \a nr in bitmap \a addr
+   \return the value of bit \a nr */
 static __inline__ int test_bit(int nr, const unsigned long *addr)
 {
         return ((1UL << (nr & (BITS_PER_LONG - 1))) &
                 ((addr)[nr / BITS_PER_LONG])) != 0;
 }
 
-/* using binary seach */
+/* __fls and __ffs use binary search */
+
+/**
+ * Find last non-zero bit in bitmask \a data
+ *
+ * \param data Bitmask
+ *
+ * \return 0 if failed to find non-zero bit, position of
+ * non-zero bit (counting from 1) otherwise
+ */
 static __inline__ unsigned long __fls(long data)
 {
 	int pos = 32;
@@ -115,6 +145,20 @@ static __inline__ unsigned long __fls(lo
 	return pos;
 }
 
+/**
+ * Find first non-zero bit in bitmask \a data
+ *
+ * \param data Bitmask
+ *
+ * \return sizeof(long)-1 if failed to find non-zero bit, position of
+ * non-zero bit (counting from 0) otherwise
+ *
+ * NOTE: the function returns the same value when:
+ * - \a data is equal to zero
+ * - \a data has the most significant bit set and other bits clear.
+ *
+ * Callers may check whether \a data == 0 to discriminate.
+ */
 static __inline__ unsigned long __ffs(long data)
 {
         int pos = 0;
@@ -147,7 +191,30 @@ static __inline__ unsigned long __ffs(lo
         return pos;
 }
 
+/**
+ * Find first zero bit in bitmask \a x
+ *
+ * \param x Bitmask
+ *
+ * \return sizeof(long)-1 if failed to find zero bit, position of
+ * zero bit (counting from 0) otherwise
+ *
+ * NOTE: the function returns the same value when:
+ * - \a x has all bits set
+ * - \a x has the most significant bit clear and other bits set.
+ *
+ * Callers may check whether \a data == ~0UL to discriminate.
+ */
 #define __ffz(x)	__ffs(~(x))
+
+/**
+ * Find last zero bit in bitmask \a x
+ *
+ * \param x Bitmask
+ *
+ * \return 0 if failed to find zero bit, position of
+ * zero bit (counting from 1) otherwise
+ */
 #define __flz(x)	__fls(~(x))
 
 unsigned long find_next_bit(unsigned long *addr,
@@ -156,7 +223,14 @@ unsigned long find_next_bit(unsigned lon
 unsigned long find_next_zero_bit(unsigned long *addr,
                                  unsigned long size, unsigned long offset);
 
+/** the same as find_next_bit but start search from the beginning of bitmask */
 #define find_first_bit(addr,size)       (find_next_bit((addr),(size),0))
+/** the same as find_next_zero_bit but start search from the beginning
+ *  of bitmask */
 #define find_first_zero_bit(addr,size)  (find_next_zero_bit((addr),(size),0))
 
+/** @} libcfs_userspace_bitops */
+/** @} libcfs_bitops */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/user-lock.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-lock.h,v
retrieving revision 1.16
diff -u -p -r1.16 user-lock.h
--- libcfs/include/libcfs/user-lock.h	15 Sep 2009 21:09:24 -0000	1.16
+++ libcfs/include/libcfs/user-lock.h	1 Jan 2010 20:03:32 -0000
@@ -45,14 +45,33 @@
 #error Do not #include this file directly. #include <libcfs/libcfs.h> instead
 #endif
 
-/* Implementations of portable synchronization APIs for liblustre */
+/** \addtogroup libcfs
+ * @{ */
 
-/*
- * liblustre is single-threaded, so most "synchronization" APIs are trivial.
+/** \addtogroup libcfs_synchronization
+ * @{ */
+
+/**
+ * \defgroup libcfs_userspace_synchronization Synchronization (user-space)
+ * User-space implementation of general synchronization primitives.
  *
- * XXX Liang: There are several branches share lnet with b_hd_newconfig,
- * if we define lock APIs at here, there will be conflict with liblustre
- * in other branches.
+ * There are two main consumers for this API: pure single-threaded liblustre
+ * (when operation environment provides only one thread of execution for all
+ * Lustre subsystems including lnet and LNDs) and single-threaded liblustre in
+ * MT environment (when lnet and LNDs enjoy multi-threadness while liblustre
+ * still runs in one particular thread).
+ *
+ * To meet the former case, user-space Libcfs implements all synchronization
+ * primitives as trivial stubs assuming that no race-conditions are possible
+ * in single-threaded case. For the latter case, it provides MT-safe
+ * implementation for some primitives (cfs_completion and cfs_atomic) that
+ * lnet and LNDs really need in MT environment.
+ *
+ * The fact that even running in MT environment some primitives are MT-safe and
+ * others are not makes the code pretty confusing and prone for misuse.
+ * Hopefully, it will be fixed soon in the course of name-normalization efforts.
+ *
+ * @{
  */
 
 #ifndef __KERNEL__
@@ -67,8 +86,9 @@
  * Optional debugging (magic stamping and checking ownership) can be added.
  */
 
-/*
- * spin_lock
+/**
+ * \defgroup libcfs_userspace_spin_locks Spin-locks
+ * spin_lock (user-space, single-threaded)
  *
  * - spin_lock_init(x)
  * - spin_lock(x)
@@ -79,6 +99,7 @@
  * - spin_unlock_irqrestore(x, f)
  *
  * No-op implementation.
+ * @{
  */
 struct spin_lock {int foo;};
 
@@ -100,13 +121,18 @@ void spin_unlock_bh(spinlock_t *lock);
 static inline int spin_is_locked(spinlock_t *l) {return 1;}
 static inline void spin_lock_irqsave(spinlock_t *l, unsigned long f){}
 static inline void spin_unlock_irqrestore(spinlock_t *l, unsigned long f){}
+/** @} */
 
-/*
- * Semaphore
+/**
+ * \defgroup libcfs_userspace_semaphore Semaphore
+ * Semaphore (user-space, single-threaded)
  *
  * - sema_init(x, v)
  * - __down(x)
  * - __up(x)
+ *
+ * No-op implementation.
+ * @{
  */
 typedef struct semaphore {
     int foo;
@@ -115,14 +141,19 @@ typedef struct semaphore {
 void sema_init(struct semaphore *s, int val);
 void __down(struct semaphore *s);
 void __up(struct semaphore *s);
+/** @} */
 
-/*
- * Mutex:
+/**
+ * \defgroup libcfs_userspace_mutex Mutex
+ * Mutex (user-space, single-threaded)
  *
  * - init_mutex(x)
  * - init_mutex_locked(x)
  * - mutex_up(x)
  * - mutex_down(x)
+ *
+ * Direct mapping to no-op semaphores
+ * @{
  */
 #define DECLARE_MUTEX(name)     \
         struct semaphore name = { 1 }
@@ -135,13 +166,36 @@ void __up(struct semaphore *s);
 #define init_MUTEX(x)                   sema_init(x, 1)
 #define init_MUTEX_LOCKED(x)            sema_init(x, 0)
 #define init_mutex(s)                   init_MUTEX(s)
+/** @} */
 
-/*
- * Completion:
+/**
+ * \defgroup libcfs_userspace_completion Completion
+ * Completion (user-space, single-threaded)
  *
  * - init_completion(c)
  * - complete(c)
  * - wait_for_completion(c)
+ *
+ * User-space single-threaded completion API is the super-set of completion API
+ * for MT-environment. The source of complexity is that we have only one thread
+ * of execution, so a subsystem that needs to wait for completion must
+ * somehow relinquish execution control to allow another subsystem succeed. And,
+ * in turn, that another subsystem generating completion event, should somehow
+ * pass execution control back to the former.
+ *
+ * Essentially, there is only one additional API call for user-space
+ * single-threaded case: \a init_completion_module. It should be called once,
+ * usually while start-up, with the only argument of \a cfs_wait_handler_t
+ * type. This argument is some caller-specific callback that will be called
+ * every time as Libcfs decides to relinquish execution control imitating sleep
+ * waiting for resource. For example, \a wait_for_completion would call this
+ * callback if completion event were not arrived yet.
+
+ * It's responsibility of caller to implement proper task scheduling inside this
+ * callback fitting its needs. See lustre/ptlrpc/events.c ::liblustre_wait_event
+ * as example.
+ *
+ * @{ 
  */
 struct completion {
         unsigned int done;
@@ -162,16 +216,20 @@ int wait_for_completion_interruptible(st
         struct completion work = COMPLETION_INITIALIZER(work)
 
 #define INIT_COMPLETION(x)      ((x).done = 0)
+/** @} */
 
-
-/*
- * rw_semaphore:
+/**
+ * \defgroup libcfs_userspace_rw_semaphore Read-write semaphores
+ * rw_semaphore (user-space, single-threaded)
  *
  * - init_rwsem(x)
  * - down_read(x)
  * - up_read(x)
  * - down_write(x)
  * - up_write(x)
+ *
+ * No-op implementation.
+ * @{
  */
 struct rw_semaphore {
         int foo;
@@ -185,10 +243,11 @@ int down_write_trylock(struct rw_semapho
 void up_read(struct rw_semaphore *s);
 void up_write(struct rw_semaphore *s);
 void fini_rwsem(struct rw_semaphore *s);
+/** @} */
 
-/*
- * read-write lock : Need to be investigated more!!
- * XXX nikita: for now, let rwlock_t to be identical to rw_semaphore
+/**
+ * \defgroup libcfs_userspace_rw_lock Read-write locks
+ * read-write lock (user-space, single-threaded)
  *
  * - DECLARE_RWLOCK(l)
  * - rwlock_init(x)
@@ -196,6 +255,9 @@ void fini_rwsem(struct rw_semaphore *s);
  * - read_unlock(x)
  * - write_lock(x)
  * - write_unlock(x)
+ *
+ * Direct mapping to no-op rw_semaphores
+ * @{
  */
 typedef struct rw_semaphore rwlock_t;
 #define RW_LOCK_UNLOCKED        (rwlock_t) { }
@@ -216,10 +278,17 @@ static inline void
 read_lock_irqsave(rwlock_t *l, unsigned long f) { read_lock(l); }
 static inline void
 read_unlock_irqrestore(rwlock_t *l, unsigned long f) { read_unlock(l); }
+/** @} */
 
-/*
- * Atomic for user-space
- * Copied from liblustre
+/**
+ * \defgroup libcfs_userspace_atomic Atomics
+ * Atomic for single-threaded user-space
+ *
+ * Trivial implementation. Copied from liblustre.
+ *
+ * NOTE: Actually, these atomics are not atomic at all. Be careful not to use
+ * them in MT environment when access is possible from more than one thread.
+ * @{
  */
 typedef struct { volatile int counter; } atomic_t;
 
@@ -239,18 +308,30 @@ typedef struct { volatile int counter; }
 #define atomic_dec_return(a)  atomic_sub_return(1,a)
 #define atomic_add_unless(v, a, u) ((v)->counter != u ? (v)->counter += a : 0)
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
-
+/** @} */
 
 #ifdef HAVE_LIBPTHREAD
 #include <pthread.h>
 
-/*
- * Completion
+/**
+ * \defgroup libcfs_userspace_mt_completion Completions (MT-safe)
+ * Completions (user-space, multi-threaded)
+ *
+ * MT-safe version of completion primitives is implemented on the top of
+ * pthread conditional variables (cond-vars). Usage is simple: the caller
+ * initialize cfs_completion with \a cfs_init_completion before spawning
+ * threads, then one thread can wait using \a cfs_wait_for_completion and
+ * another thread can wake-up the former using \a cfs_complete. That's it.
+ * @{
  */
 
+/** MT-safe verision of completion structure */
 struct cfs_completion {
+        /** flag - completion event happened */
         int c_done;
+        /** cond-var to sleep on */
         pthread_cond_t c_cond;
+        /** mutex to protect c_cond and c_done */
         pthread_mutex_t c_mut;
 };
 
@@ -258,9 +339,16 @@ void cfs_init_completion(struct cfs_comp
 void cfs_fini_completion(struct cfs_completion *c);
 void cfs_complete(struct cfs_completion *c);
 void cfs_wait_for_completion(struct cfs_completion *c);
+/** @} */
 
-/*
- * atomic.h
+/**
+ * \defgroup libcfs_userspace_mt_atomic Atomics (MT-safe)
+ * Atomics (user-space, multi-threaded)
+ *
+ * MT-safe version of atomic primitives is implemented using
+ * global pthread mutex protecting operations on
+ * cfs_atomic_t::counter.
+ * @{
  */
 
 typedef struct { volatile int counter; } cfs_atomic_t;
@@ -272,9 +360,13 @@ void cfs_atomic_inc(cfs_atomic_t *a);
 void cfs_atomic_dec(cfs_atomic_t *a);
 void cfs_atomic_add(int b, cfs_atomic_t *a);
 void cfs_atomic_sub(int b, cfs_atomic_t *a);
+/** @} */
 
 #endif /* HAVE_LIBPTHREAD */
 
+/** \addtogroup libcfs_userspace_mutex
+ * @{ */
+
 /**************************************************************************
  *
  * Mutex interface.
@@ -332,7 +424,7 @@ static inline void mutex_destroy(struct 
 {
 }
 
-/*
+/**
  * This is for use in assertions _only_, i.e., this function should always
  * return 1.
  *
@@ -345,13 +437,21 @@ static inline int mutex_is_locked(struct
         return 1;
 }
 
+/** @} libcfs_userspace_mutex */
 
 /**************************************************************************
  *
  * Lockdep "implementation". Also see lustre_compat25.h
  *
  **************************************************************************/
-
+/**
+ * \defgroup libcfs_userspace_lockdep Lockdep
+ * Dummy stubs imitating linux 'Lock-class' validator
+ *
+ * Needed only to compile some lustre code in both kernel- and user-space
+ * avoiding lengthy "#ifdef HAVE_LOCK_CLASS ... #endif" constructions.
+ * @{
+ */
 struct lock_class_key {
         int foo;
 };
@@ -368,17 +468,19 @@ static inline void lockdep_on(void)
 {
 }
 
-/* This has to be a macro, so that can be undefined in kernels that do not
- * support lockdep. */
 #define mutex_lock_nested(mutex, subclass) mutex_lock(mutex)
 #define spin_lock_nested(lock, subclass) spin_lock(lock)
 #define down_read_nested(lock, subclass) down_read(lock)
 #define down_write_nested(lock, subclass) down_write(lock)
-
+/** @} */
 
 /* !__KERNEL__ */
 #endif
 
+/** @} libcfs_userspace_synchronization */
+/** @} libcfs_synchronization */
+/** @} libcfs */
+
 /* __LIBCFS_USER_LOCK_H__ */
 #endif
 /*
Index: libcfs/include/libcfs/user-mem.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-mem.h,v
retrieving revision 1.4
diff -u -p -r1.4 user-mem.h
--- libcfs/include/libcfs/user-mem.h	4 Feb 2009 01:45:47 -0000	1.4
+++ libcfs/include/libcfs/user-mem.h	1 Jan 2010 20:03:32 -0000
@@ -9,13 +9,31 @@
 #error "This is only for user space."
 #endif
 
+/**
+ * \addtogroup libcfs
+ * @{
+ *
+ * \addtogroup libcfs_mem
+ * @{
+ *
+ * \defgroup libcfs_user_mem Memory Allocations (user-space)
+ * User-space implementation of API for memory allocations
+ * @{
+ */
 
 /* XXX
  * for this moment, liblusre will not rely OST for non-page-aligned write
  */
 #define LIBLUSTRE_HANDLE_UNALIGNED_PAGE
 
+/**
+ * User-space emulation of kernel page structure
+ *
+ * The only field of wide use is \a addr. All others are reserved for
+ * internal use in liblustre file i/o.
+ */
 typedef struct page {
+        /** CPU vaddr to access memory associated with this page */
         void   *addr;
         unsigned long index;
         struct list_head list;
@@ -70,11 +88,12 @@ static inline void *cfs_alloc(size_t nr_
 #define cfs_free_large(addr) cfs_free(addr)
 
 #define CFS_ALLOC_ATOMIC_TRY   (0)
-/*
- * SLAB allocator
+/**
+ * User-space SLAB allocator
  */
 typedef struct {
-         int size;
+        /** size of objects in this cache */
+        int size;
 } cfs_mem_cache_t;
 
 #define SLAB_HWCACHE_ALIGN 0
@@ -89,6 +108,9 @@ void *cfs_mem_cache_alloc(cfs_mem_cache_
 void cfs_mem_cache_free(cfs_mem_cache_t *c, void *addr);
 int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem);
 
+/** @} libcfs_user_mem */
+/** @} libcfs_mem */
+
 /*
  * Copy to/from user
  */
@@ -104,4 +126,5 @@ static inline int copy_to_user(void *a,v
         return 0;
 }
 
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/user-prim.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-prim.h,v
retrieving revision 1.8
diff -u -p -r1.8 user-prim.h
--- libcfs/include/libcfs/user-prim.h	1 Dec 2009 02:23:43 -0000	1.8
+++ libcfs/include/libcfs/user-prim.h	1 Jan 2010 20:03:32 -0000
@@ -45,6 +45,9 @@
 #error Do not #include this file directly. #include <libcfs/libcfs.h> instead
 #endif
 
+/** \addtogroup libcfs
+ * @{ */
+
 /* Implementations of portable APIs for liblustre */
 
 /*
@@ -53,8 +56,65 @@
 
 #ifndef __KERNEL__
 
+/* XXX Maxim: it's totally unclear why we need lproc stuff in user-space libcfs
+ * headers. Every libcfs consumer should embrace all lproc-related bits with
+ * #ifdef LPROCFS ... #endif and ensure that LPROCFS undefined in user-space
+ * builds. */
+
+/** \addtogroup libcfs_procfs
+ * @{
+ * \defgroup libcfs_user_procfs Procfs (user-space)
+ * User-space implementation of procfs API
+ *
+ * Libcfs doesn't support manipulating procfs entries and sysctl tables in
+ * user-space. It only provides stubs for some procfs-related types:
+ *
+ * - cfs_proc_dir_entry_t
+ * - cfs_read_proc_t
+ * - cfs_write_proc_t
+ *
+ * and for pseudo-device type and operations:
+ *
+ * - cfs_psdev_t
+ * - cfs_psdev_register
+ * - cfs_psdev_deregister
+ *
+ * Libcfs also abstracts file operations for dealing with /proc entries and /dev
+ * devices from user-space to allow winnt-specific libcfs to implement them in a
+ * subtle way hiding winnt peculiarities from callers (cfs_proc_* primitives).
+ * @{
+ */
 typedef struct proc_dir_entry           cfs_proc_dir_entry_t;
 
+typedef int (cfs_read_proc_t)(char *page, char **start, off_t off,
+                          int count, int *eof, void *data);
+
+struct file; /* forward ref */
+typedef int (cfs_write_proc_t)(struct file *file, const char *buffer,
+                               unsigned long count, void *data);
+
+/* proc interface wrappers for non-win OS */
+#ifndef __WINNT__
+#define cfs_proc_open   open
+#define cfs_proc_mknod  mknod
+#define cfs_proc_ioctl  ioctl
+#define cfs_proc_close  close
+#define cfs_proc_read   read
+#define cfs_proc_write  write
+#define cfs_proc_fopen  fopen
+#define cfs_proc_fclose fclose
+#define cfs_proc_fgets  fgets
+/* !__WINNT__ */
+#endif
+
+/** @} libcfs_user_procfs */
+/** @} libcfs_procfs */
+
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_userspace_synchronization
+ * @{ */
+
 /*
  * Just present a single processor until will add thread support.
  */
@@ -68,8 +128,17 @@ typedef struct proc_dir_entry           
 #define num_possible_cpus() 1
 #endif
 
-/*
- * Wait Queue. 
+/**
+ * \defgroup libcfs_userspace_waitqueue Wait queues
+ * wait-queue (user-space, single-threaded)
+ *
+ * The most part of primitives is no-ops. The only exceptions are
+ * cfs_waitq_wait and cfs_waitq_timedwait that try to relinquish CPU
+ * calling global cfs_wait_handler_t callback (see
+ * \ref libcfs_userspace_completion for details).
+ *
+ * \see \ref libcfs_waitqueue for general description
+ * @{
  */
 
 typedef struct cfs_waitlink {
@@ -81,7 +150,7 @@ typedef struct cfs_waitq {
         struct list_head sleepers;
 } cfs_waitq_t;
 
-/* XXX: need to replace wake_up with cfs_waitq_signal() */
+/** XXX: need to replace wake_up with cfs_waitq_signal() */
 #define wake_up(q) cfs_waitq_signal(q)
 
 /*
@@ -93,36 +162,50 @@ typedef long cfs_task_state_t;
 #define CFS_TASK_UNINT          (1)
 #define CFS_TASK_RUNNING        (2)
 
-
-/* 
- * Lproc
- */
-typedef int (cfs_read_proc_t)(char *page, char **start, off_t off,
-                          int count, int *eof, void *data);
-
-struct file; /* forward ref */
-typedef int (cfs_write_proc_t)(struct file *file, const char *buffer,
-                               unsigned long count, void *data);
-
-/*
- * Signal
+/** @} libcfs_userspace_waitqueue */
+/** @} libcfs_userspace_synchronization */
+/** @} libcfs_synchronization */
+
+/** \addtogroup libcfs_signal
+ * @{
+ * \defgroup libcfs_userspace_signal Signal (user-space)
+ * Implementation of user-space signal API is based on sigprocmask(2) and
+ * sigpending(2), daemonization primitives are no-ops.
+ * @{
  */
 typedef sigset_t                        cfs_sigset_t;
 
-/*
- * Timer
- */
+#define cfs_sigfillset(l) do {}         while (0)
+#define cfs_recalc_sigpending(l)        do {} while (0)
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_timer
+ * @{
+ *
+ * \defgroup libcfs_userspace_timer Timer (user-space)
+ * Timer (user-space, single-threaded)
+ *
+ * Be careful: cfs_timer_t looks sound and API does something but it's
+ * illusion - nobody ever calls timer callback! Regard it as no-op
+ * implementation.
+ * @{
+ */
 typedef struct {
         struct list_head tl_list;
         void (*function)(ulong_ptr_t unused);
         ulong_ptr_t data;
         long expires;
 } cfs_timer_t;
-
+/** @} */
+/** @} */
 
 #define in_interrupt()    (0)
 
+/** \addtogroup libcfs_ioctl
+ * @{
+ * \addtogroup libcfs_userspace_ioctl
+ * @{ */
 typedef void cfs_psdev_t;
 
 static inline int cfs_psdev_register(cfs_psdev_t *foo)
@@ -134,10 +217,10 @@ static inline int cfs_psdev_deregister(c
 {
         return 0;
 }
+/** @} */
+/** @} */
 
 #define cfs_lock_kernel()               do {} while (0)
-#define cfs_sigfillset(l) do {}         while (0)
-#define cfs_recalc_sigpending(l)        do {} while (0)
 #define cfs_kernel_thread(l,m,n)        LBUG()
 #define cfs_kthread_run(fn,d,fmt,...)   LBUG()
 
@@ -148,23 +231,58 @@ int cfs_create_thread(cfs_thread_t func,
 #define cfs_create_thread(l,m) LBUG()
 #endif
 
+/** \addtogroup libcfs_curproc
+ * @{
+ * \addtogroup libcfs_userspace_curproc
+ * @{ */
 uid_t cfs_curproc_uid(void);
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_mem
+ * @{
+ * \addtogroup libcfs_user_mem
+ * @{ */
 #define LIBCFS_REALLOC(ptr, size) realloc(ptr, size)
+/** @} */
+/** @} */
 
 #define cfs_online_cpus() sysconf(_SC_NPROCESSORS_ONLN)
 
 // static inline void local_irq_save(unsigned long flag) {return;}
 // static inline void local_irq_restore(unsigned long flag) {return;}
 
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_stack_trace
+ * @{
+ * \defgroup libcfs_stack_trace_userspace Stack-tracing (user-space)
+ * User-space implementation of stack-tracing
+ * @{
+ */
+
 enum {
+        /** maximum # frames in stack-trace */
         CFS_STACK_TRACE_DEPTH = 16
 };
 
+/**
+ * User-space version of stack-trace structure.
+ * Plain array of opaque pointers to stack-trace frames.
+ */
 struct cfs_stack_trace {
         void *frame[CFS_STACK_TRACE_DEPTH];
 };
 
+/** @} libcfs_stack_trace_userspace */
+/** @} libcfs_stack_trace */
+/** @} libcfs_debug */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_userspace_misc
+ * @{ */
+
 /*
  * arithmetic
  */
@@ -199,23 +317,14 @@ static inline void libcfs_arch_cleanup(v
 /* __WINNT__ */
 #endif
 
-/* proc interface wrappers for non-win OS */
-#ifndef __WINNT__
-#define cfs_proc_open   open
-#define cfs_proc_mknod  mknod
-#define cfs_proc_ioctl  ioctl
-#define cfs_proc_close  close
-#define cfs_proc_read   read
-#define cfs_proc_write  write
-#define cfs_proc_fopen  fopen
-#define cfs_proc_fclose fclose
-#define cfs_proc_fgets  fgets
-/* !__WINNT__ */
-#endif
+/** @} libcfs_userspace_misc */
+/** @} libcfs_misc */
 
 /* !__KERNEL__ */
 #endif
 
+/** @} libcfs */
+
 /* __LIBCFS_USER_PRIM_H__ */
 #endif
 /*
Index: libcfs/include/libcfs/user-tcpip.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-tcpip.h,v
retrieving revision 1.5
diff -u -p -r1.5 user-tcpip.h
--- libcfs/include/libcfs/user-tcpip.h	3 Feb 2009 13:43:17 -0000	1.5
+++ libcfs/include/libcfs/user-tcpip.h	1 Jan 2010 20:03:32 -0000
@@ -45,10 +45,25 @@
 
 #include <sys/uio.h>
 
+/**
+ * \addtogroup libcfs
+ * @{
+ *
+ * \addtogroup libcfs_net
+ * @{
+ *
+ * \defgroup libcfs_user_net Network API (user-space)
+ *
+ * See \ref libcfs_net for generic description.
+ * @{
+ */
+
 /*
  * Functions to get network interfaces info
  */
 
+/* XXX Maxim: libcfs_sock_ioctl is not part of libcfs API. Move its
+ * prototype to user-tcpip.c */
 int libcfs_sock_ioctl(int cmd, unsigned long arg);
 int libcfs_ipif_query (char *name, int *up, __u32 *ip);
 void libcfs_ipif_free_enumeration (char **names, int n);
@@ -58,10 +73,13 @@ int libcfs_ipif_enumerate (char ***names
  * Network function used by user-land lnet acceptor
  */
 
+/** This is how user-space libcfs abstracts socket file descriptor */
 typedef struct cfs_socket {
+        /** file descriptor acquired with socket(2) */
         int s_fd;
 } cfs_socket_t;
 
+/** usocklnd needs it to poll(2) */
 #define LIBCFS_SOCK2FD(sock) ((sock)->s_fd)
 
 int libcfs_sock_listen(cfs_socket_t **sockp, __u32 ip, int port, int backlog);
@@ -111,6 +129,9 @@ int libcfs_sock_create(cfs_socket_t **so
 #error "Undefined byteorder??"
 #endif /* __LITTLE_ENDIAN */
 
+/** @} libcfs_user_net */
+/** @} libcfs_net */
+/** @} libcfs */
 #endif /* !__KERNEL__ */
 
 #endif
Index: libcfs/include/libcfs/user-time.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/user-time.h,v
retrieving revision 1.5
diff -u -p -r1.5 user-time.h
--- libcfs/include/libcfs/user-time.h	7 Oct 2008 04:14:07 -0000	1.5
+++ libcfs/include/libcfs/user-time.h	1 Jan 2010 20:03:32 -0000
@@ -47,9 +47,16 @@
 #error Do not #include this file directly. #include <libcfs/libcfs.h> instead
 #endif
 
-/* Portable time API */
+/** \addtogroup libcfs
+ * @{ */
 
-/*
+/** \addtogroup libcfs_time
+ * @{ */
+
+/**
+ * \defgroup libcfs_user_time Time (user-space)
+ * Portable time API for user-space
+ *
  * Platform provides three opaque data-types:
  *
  *  cfs_time_t        represents point in time. This is internal kernel
@@ -83,7 +90,7 @@
  *
  *  CFS_TIME_FORMAT
  *  CFS_DURATION_FORMAT
- *
+ * @{
  */
 
 #ifndef __KERNEL__
@@ -95,8 +102,11 @@
  * Liblustre. time(2) based implementation.
  */
 
+/** time(2) based implementation */
 typedef time_t cfs_fs_time_t;
+/** time(2) based implementation */
 typedef time_t cfs_time_t;
+/** time(2) based implementation */
 typedef time_t cfs_duration_t;
 
 /* looks like linux */
@@ -192,6 +202,10 @@ static inline int cfs_fs_time_beforeq(cf
 /* !__KERNEL__ */
 #endif
 
+/** @} libcfs_user_time */
+/** @} libcfs_time */
+/** @} libcfs */
+
 /* __LIBCFS_USER_TIME_H__ */
 #endif
 /*
Index: libcfs/include/libcfs/linux/kp30.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/kp30.h,v
retrieving revision 1.15
diff -u -p -r1.15 kp30.h
--- libcfs/include/libcfs/linux/kp30.h	26 Aug 2009 09:41:26 -0000	1.15
+++ libcfs/include/libcfs/linux/kp30.h	1 Jan 2010 20:03:33 -0000
@@ -80,6 +80,9 @@
 
 #include <libcfs/linux/portals_compat25.h>
 
+/** \addtogroup libcfs
+ * @{ */
+
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 #define schedule_work schedule_task
 #define prepare_work(wq,cb,cbdata)                                            \
@@ -131,6 +134,14 @@ static inline void our_cond_resched(void
 
 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0) */
 
+
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_linux_synchronization
+ * @{ */
+
+/** \addtogroup libcfs_linux_spin_locks
+ * @{ */
 #ifdef CONFIG_SMP
 #define LASSERT_SPIN_LOCKED(lock) LASSERT(spin_is_locked(lock))
 #define LINVRNT_SPIN_LOCKED(lock) LINVRNT(spin_is_locked(lock))
@@ -138,6 +149,10 @@ static inline void our_cond_resched(void
 #define LASSERT_SPIN_LOCKED(lock) do {(void)sizeof(lock);} while(0)
 #define LINVRNT_SPIN_LOCKED(lock) do {(void)sizeof(lock);} while(0)
 #endif
+/** @} libcfs_linux_spin_locks */
+
+/** \defgroup libcfs_linux_semaphore Semaphore
+ * @{ */
 #define LASSERT_SEM_LOCKED(sem) LASSERT(down_trylock(sem) != 0)
 
 #ifdef HAVE_SEM_COUNT_ATOMIC
@@ -145,11 +160,26 @@ static inline void our_cond_resched(void
 #else
 #define SEM_COUNT(sem)          ((sem)->count)
 #endif
+/** @} libcfs_linux_semaphore */
 
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_linux_debug
+ * @{ */
 #define LIBCFS_PANIC(msg)            panic(msg)
+/** @} */
+/** @} */
 
 /* ------------------------------------------------------------------- */
 
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
+
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 
 #define PORTAL_SYMBOL_REGISTER(x) inter_module_register(#x, THIS_MODULE, &x)
@@ -193,6 +223,9 @@ static inline void our_cond_resched(void
 #endif
 /******************************************************************************/
 
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
+
 #if (__GNUC__)
 /* Use the special GNU C __attribute__ hack to have the compiler check the
  * printf style argument string against the actual argument count and
@@ -205,21 +238,46 @@ static inline void our_cond_resched(void
 
 #endif /* __GNUC__ */
 
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_linux_debug
+ * @{ */
 # define fprintf(a, format, b...) CDEBUG(D_OTHER, format , ## b)
 # define printf(format, b...) CDEBUG(D_OTHER, format , ## b)
+/** @} libcfs_linux_debug */
+/** @} libcfs_debug */
+
+/** \addtogroup libcfs_time
+ * @{
+ * \addtogroup libcfs_linux_time
+ * @{ */
 # define time(a) CURRENT_TIME
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_linux_synchronization
+ * @{ */
 #ifndef num_possible_cpus
 #define num_possible_cpus() NR_CPUS
 #endif
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
 #define i_size_read(a) ((a)->i_size)
 #endif
 
 
-/******************************************************************************/
-/* Light-weight trace
- * Support for temporary event tracing with minimal Heisenberg effect. */
+/**
+ * \addtogroup libcfs_lwt
+ * @{
+ * \defgroup libcfs_linux_lwt Light-weight tracing (linux-specific)
+ *
+ * Linux implementation of LWT. See \ref libcfs_lwt for general description.
+ * @{
+ */
 #define LWT_SUPPORT  0
 
 #define LWT_MEMORY   (16<<20)
@@ -322,10 +380,23 @@ extern int  lwt_snapshot (cycles_t *now,
 #  define LWT_EVENT(p1,p2,p3,p4)     /* no userland implementation yet */
 # endif /* __KERNEL__ */
 #endif /* LWT_SUPPORT */
+/** @} libcfs_lwt */
+/** @} libcfs_linux_lwt */
 
 /* ------------------------------------------------------------------ */
 
+/** \addtogroup libcfs_ioctl
+ * @{
+ * \addtogroup libcfs_linux_ioctl
+ * @{ */
 #define IOCTL_LIBCFS_TYPE long
+/** @} */
+/** @} */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
 
 #ifdef __CYGWIN__
 # ifndef BITS_PER_LONG
@@ -405,6 +476,14 @@ static inline void sg_set_page(struct sc
 }
 #endif
 
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
+
+/** \addtogroup libcfs_schedule
+ * @{
+ * \defgroup libcfs_linux_schedule Schedule (linux-specific)
+ * @{ */
+
 #ifndef get_cpu
 # ifdef CONFIG_PREEMPT
 #  define cfs_get_cpu()  ({ preempt_disable(); smp_processor_id(); })
@@ -418,4 +497,9 @@ static inline void sg_set_page(struct sc
 # define cfs_put_cpu()   put_cpu()
 #endif /* get_cpu & put_cpu */
 
+/** @} libcfs_linux_schedule */
+/** @} libcfs_schedule */
+
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/linux/libcfs.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/libcfs.h,v
retrieving revision 1.11
diff -u -p -r1.11 libcfs.h
--- libcfs/include/libcfs/linux/libcfs.h	9 Dec 2009 16:18:35 -0000	1.11
+++ libcfs/include/libcfs/linux/libcfs.h	1 Jan 2010 20:03:33 -0000
@@ -62,6 +62,24 @@
 #include <asm/timex.h>
 #include <linux/sched.h> /* THREAD_SIZE */
 
+/** \addtogroup libcfs
+ * @{ */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \defgroup libcfs_linux_debug Debugging (linux-specific, kernel-space)
+ * Linux-specific implementation of Libcfs debug primitives for
+ * in-kernel users.
+ *
+ * On some platforms (e.g. i386) the stack is very limited resource
+ * for kernel threads. Libcfs provides the following primitives to
+ * control its usage:
+ * - CDEBUG_STACK - calculates current depth of stack
+ * - CHECK_STACK - print out warning message if the stack grew higher
+ * than the waterline (3/4 of THREAD_SIZE).
+ * @{
+ */
+
 #define LUSTRE_TRACE_SIZE (THREAD_SIZE >> 5)
 
 #if !defined(__x86_64__)
@@ -75,6 +93,7 @@
                            (THREAD_SIZE - 1)))
 # endif /* __ia64__ */
 
+/** print out warning message if the stack grew high */
 #define __CHECK_STACK(file, func, line)                                 \
 do {                                                                    \
         unsigned long _stack = CDEBUG_STACK();                           \
@@ -94,36 +113,94 @@ do {                                    
 #define CDEBUG_STACK() (0L)
 #endif /* __x86_64__ */
 
-/* initial pid  */
+/** @} libcfs_linux_debug */
+/** @} libcfs_debug */
+
+/* XXX Maxim: We shouldn't duplicate defining LUSTRE_LNET_PID in all
+ * platform-specific libcfs.h headers. Move it to common libcfs.h. */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
+/** initial pid  */
 #define LUSTRE_LNET_PID          12345
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_linux_debug
+ * @{ */
 #define ENTRY_NESTING_SUPPORT (1)
 #define ENTRY_NESTING   do {;} while (0)
 #define EXIT_NESTING   do {;} while (0)
 #define __current_nesting_level() (0)
+/** @} libcfs_debug */
+/** @} libcfs_linux_debug */
+
+/** \addtogroup libcfs_curproc
+ * @{
+ * \addtogroup libcfs_kernel_curproc
+ * @{ */
 
 /**
- * Platform specific declarations for cfs_curproc API (libcfs/curproc.h)
+ * Linux specific declarations for cfs_curproc API (libcfs/curproc.h)
  *
  * Implementation is in linux-curproc.c
  */
+
 #define CFS_CURPROC_COMM_MAX (sizeof ((struct task_struct *)0)->comm)
 
 #include <linux/capability.h>
 typedef kernel_cap_t cfs_kernel_cap_t;
 
-/*
+/** @} libcfs_kernel_curproc */
+/** @} libcfs_curproc */
+
+/** \addtogroup libcfs_debug
+ * @{
+ * \addtogroup libcfs_stack_trace
+ * @{
+ * \defgroup libcfs_stack_trace_linux Stack-tracing (linux-specific)
+ * Linux-specific implementation of stack-tracing
+ * @{
+ */
+
+/**
  * No stack-back-tracing in Linux for now.
  */
 struct cfs_stack_trace {
 };
 
+/** @} libcfs_stack_trace_linux */
+/** @} libcfs_stack_trace */
+/** @} libcfs_debug */
+
+
+/** \addtogroup libcfs_types
+ * @{
+ *\defgroup libcfs_linux_types Generic types (linux-specific, kernel-space)
+ *
+ * Definitions of some generic types for linux.
+ *
+ * Most generic types and all primitives for swabbing are already
+ * defined in linux system headers. Libcfs only adds definitions for
+ * long and unsigned long integers with size equal to pointer.
+ * @{
+ */
+
 /* long integer with size equal to pointer */
 typedef unsigned long ulong_ptr_t;
 typedef long long_ptr_t;
 
+/** @} libcfs_linux_types */
+/** @} libcfs_types */
+
 #ifndef WITH_WATCHDOG
 #define WITH_WATCHDOG
 #endif
 
+/** @} libcfs */
+
 #endif /* _LINUX_LIBCFS_H */
Index: libcfs/include/libcfs/linux/linux-fs.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-fs.h,v
retrieving revision 1.6
diff -u -p -r1.6 linux-fs.h
--- libcfs/include/libcfs/linux/linux-fs.h	23 Jun 2009 20:59:03 -0000	1.6
+++ libcfs/include/libcfs/linux/linux-fs.h	1 Jan 2010 20:03:33 -0000
@@ -53,6 +53,16 @@
 #include <linux/stat.h>
 #include <linux/mount.h>
 
+/**
+ * \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_fs
+ * @{
+ * \defgroup libcfs_linux_fs File operations for kernel users (linux-specific)
+ * Linux implementation of Libcfs API for manipulating files in kernel
+ * @{
+ */
+
 typedef struct file cfs_file_t;
 typedef struct dentry cfs_dentry_t;
 typedef struct dirent64 cfs_dirent_t;
@@ -90,4 +100,8 @@ ssize_t cfs_user_write (cfs_file_t *filp
 #define CFS_IFTODT(type)           (((type) & S_IFMT) >> CFS_IFSHIFT)
 #define CFS_DTTOIF(dirtype)        ((dirtype) << CFS_IFSHIFT)
 
+/** @} libcfs_linux_fs */
+/** @} libcfs_fs */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/linux/linux-lock.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-lock.h,v
retrieving revision 1.14
diff -u -p -r1.14 linux-lock.h
--- libcfs/include/libcfs/linux/linux-lock.h	29 Nov 2008 13:12:47 -0000	1.14
+++ libcfs/include/libcfs/linux/linux-lock.h	1 Jan 2010 20:03:33 -0000
@@ -52,16 +52,36 @@
 #include <linux/smp_lock.h>
 #include <linux/mutex.h>
 
-/*
+/** \addtogroup libcfs
+ * @{ */
+
+/**
+ * \defgroup libcfs_synchronization Synchronization
+ * Generic synchronization primitives.
+ *
+ * An abstraction layer over platform-specific synchronization
+ * facilities (mutexes, semaphores, spin-locks, etc.) for protection of
+ * critical sections and facilities (like completions and wait-queues)
+ * for reliable event notifications in SMP environment. Atomic types
+ * and primitives are also in this class.
+ * @{
+ */
+
+/**
+ * \defgroup libcfs_linux_synchronization Synchronization (linux-specific)
+ * Linux-specific implementation of general synchronization primitives.
+ *
  * IMPORTANT !!!!!!!!
  *
  * All locks' declaration are not guaranteed to be initialized,
- * Althought some of they are initialized in Linux. All locks
+ * Although some of they are initialized in Linux. All locks
  * declared by CFS_DECL_* should be initialized explicitly.
+ * @{
  */
 
 
-/*
+/**
+ * \defgroup libcfs_linux_spin_locks Spin-locks
  * spin_lock (use Linux kernel's primitives)
  *
  * - spin_lock_init(x)
@@ -73,7 +93,8 @@
  * - spin_unlock_irqrestore(x, f)
  */
 
-/*
+/**
+ * \defgroup libcfs_linux_rw_sema Read-write semaphores
  * rw_semaphore (use Linux kernel's primitives)
  *
  * - init_rwsem(x)
@@ -81,10 +102,13 @@
  * - up_read(x)
  * - down_write(x)
  * - up_write(x)
+ * @{
  */
 #define fini_rwsem(s) do {} while(0)
+/** @} */
 
-/*
+/**
+ * \defgroup libcfs_linux_rw_lock Read-write locks
  * rwlock_t (use Linux kernel's primitives)
  *
  * - rwlock_init(x)
@@ -94,30 +118,37 @@
  * - write_unlock(x)
  */
 
-/*
+/**
+ * \defgroup libcfs_linux_mutex Mutex
  * mutex:
  *
  * - init_mutex(x)
  * - init_mutex_locked(x)
  * - mutex_up(x)
  * - mutex_down(x)
+ *  @{
  */
 #define init_mutex(x)                   init_MUTEX(x)
 #define init_mutex_locked(x)            init_MUTEX_LOCKED(x)
 #define mutex_up(x)                     up(x)
 #define mutex_down(x)                   down(x)
 #define mutex_down_trylock(x)           down_trylock(x)
+/** @} */
 
-/*
+/**
+ * \defgroup libcfs_linux_completion Completion
  * completion (use Linux kernel's primitives)
  *
- * - init_complition(c)
+ * - init_completion(c)
  * - complete(c)
  * - wait_for_completion(c)
  */
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,16)
 
+/** \addtogroup libcfs_linux_mutex
+ * @{ */
+
 /**************************************************************************
  *
  * Mutex interface from newer Linux kernels.
@@ -132,7 +163,7 @@ static inline void mutex_destroy(struct 
 {
 }
 
-/*
+/**
  * This is for use in assertions _only_, i.e., this function should always
  * return 1.
  *
@@ -144,8 +175,25 @@ static inline int mutex_is_locked(struct
 {
         return 1;
 }
+/** @} */
 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,16) */
 
+/**
+ * \defgroup libcfs_linux_lockdep Lockdep
+ * Linux 'Lock-class' validator
+ *
+ * 'Lock-class' validator is complex linux kernel subsystem that
+ * keeps track the sequence of locks acquired to ensure hard- and
+ * soft-irq safety, avoidance of dead-locks and correctness of lock
+ * hierarchies (when nested versions of locks prims are used). See linux
+ * kernel documentation for more details.
+ *
+ * The code below just provides dummy stubs for linux kernels that
+ * doesn't provide 'Lock-class' facility.
+ *
+ * @{
+ */
+
 #ifndef lockdep_set_class
 
 /**************************************************************************
@@ -191,7 +239,12 @@ static inline void lockdep_on(void)
 # define down_write_nested(lock, subclass) down_write(lock)
 #endif
 #endif /* CONFIG_DEBUG_LOCK_ALLOC */
+/** @} libcfs_linux_lockdep */
 
+/**
+ * \addtogroup libcfs_linux_spin_locks
+ * @{
+ */
 
 /*
  * spinlock "implementation"
@@ -204,6 +257,12 @@ typedef spinlock_t cfs_spinlock_t;
 #define cfs_spin_lock_bh(lock)   spin_lock_bh(lock)
 #define cfs_spin_unlock(lock)    spin_unlock(lock)
 #define cfs_spin_unlock_bh(lock) spin_unlock_bh(lock)
+/** @} */
+
+/**
+ * \addtogroup libcfs_linux_rw_lock
+ * @{
+ */
 
 /*
  * rwlock "implementation"
@@ -216,5 +275,10 @@ typedef rwlock_t cfs_rwlock_t;
 #define cfs_read_unlock(lock)      read_unlock(lock)
 #define cfs_write_lock_bh(lock)    write_lock_bh(lock)
 #define cfs_write_unlock_bh(lock)  write_unlock_bh(lock)
+/** @} */
+
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
+/** @} libcfs */
 
 #endif /* __LIBCFS_LINUX_CFS_LOCK_H__ */
Index: libcfs/include/libcfs/linux/linux-mem.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-mem.h,v
retrieving revision 1.7
diff -u -p -r1.7 linux-mem.h
--- libcfs/include/libcfs/linux/linux-mem.h	24 Sep 2009 15:11:09 -0000	1.7
+++ libcfs/include/libcfs/linux/linux-mem.h	1 Jan 2010 20:03:33 -0000
@@ -57,6 +57,22 @@
 # include <linux/mm_inline.h>
 #endif
 
+/**
+ * \addtogroup libcfs
+ * @{
+ *
+ * \addtogroup libcfs_mem
+ * @{
+ *
+ * \defgroup libcfs_kernel_mem Memory Allocations (kernel-space)
+ * Kernel-space implementation of API for memory allocations
+ * @{
+ *
+ * \defgroup libcfs_linux_mem Memory Allocations (kernel-space, linux-specific)
+ * Linux-specific kernel-space implementation of API for memory allocations
+ * @{
+ */
+
 typedef struct page                     cfs_page_t;
 #define CFS_PAGE_SIZE                   PAGE_CACHE_SIZE
 #define CFS_PAGE_SHIFT                  PAGE_CACHE_SHIFT
@@ -115,7 +131,7 @@ extern void __cfs_free_pages(cfs_page_t 
 #define libcfs_memory_pressure_clr() do { current->flags &= ~PF_MEMALLOC; } while (0)
 
 #if BITS_PER_LONG == 32
-/* limit to lowmem on 32-bit systems */
+/** limit to lowmem on 32-bit systems */
 #define CFS_NUM_CACHEPAGES min(num_physpages, 1UL << (30-CFS_PAGE_SHIFT) *3/4)
 #else
 #define CFS_NUM_CACHEPAGES num_physpages
@@ -142,10 +158,25 @@ extern void *cfs_mem_cache_alloc ( cfs_m
 extern void cfs_mem_cache_free ( cfs_mem_cache_t *, void *);
 extern int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem);
 
+/** @} libcfs_linux_mem */
+/** @} libcfs_kernel_mem */
+/** @} libcfs_mem */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
+
 /*
+ * When in-kernel user needs access to the memory mappped in
+ * user-space, the following macros may be used:
  */
 #define CFS_DECL_MMSPACE                mm_segment_t __oldfs
 #define CFS_MMSPACE_OPEN                do { __oldfs = get_fs(); set_fs(get_ds());} while(0)
 #define CFS_MMSPACE_CLOSE               set_fs(__oldfs)
 
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
+
+/** @} libcfs */
 #endif /* __LINUX_CFS_MEM_H__ */
Index: libcfs/include/libcfs/linux/linux-prim.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-prim.h,v
retrieving revision 1.10
diff -u -p -r1.10 linux-prim.h
--- libcfs/include/libcfs/linux/linux-prim.h	1 Dec 2009 02:23:48 -0000	1.10
+++ libcfs/include/libcfs/linux/linux-prim.h	1 Jan 2010 20:03:33 -0000
@@ -69,14 +69,58 @@
 
 #include <libcfs/linux/linux-time.h>
 
+/** \addtogroup libcfs
+ * @{ */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \defgroup libcfs_linux_misc Misc. (linux-specific)
+ * @{ */
 #define CFS_EXPORT_SYMBOL(s)            EXPORT_SYMBOL(s)
+/** @} */
+/** @} */
 
-/*
- * Pseudo device register
- */
+/** \addtogroup libcfs_ioctl
+ * @{
+ * \defgroup libcfs_linux_ioctl Pseudo devices and ioctls (linux-specific, kernel-space)
+ * @{ */
 typedef struct miscdevice		cfs_psdev_t;
 #define cfs_psdev_register(dev)		misc_register(dev)
 #define cfs_psdev_deregister(dev)	misc_deregister(dev)
+/** @} */
+/** @} */
+
+
+/** \addtogroup libcfs_procfs
+ * @{
+ * \defgroup libcfs_linux_procfs Procfs and sysctl (linux-specific, kernel-space)
+ * Linux implementation of procfs and sysctl API
+ *
+ * Generic API for procfs and sysctl is described in \ref libcfs_procfs.
+ * Libcfs implements this API for linux in-kernel users in
+ * straightforward way using appropriate linux-native types and
+ * functions.
+ *
+ * As for sysctl, it also adds another useful macro: \a DECLARE_PROC_HANDLER.
+ * So, linux-specific consumer code has two ways for defining sysctl callback:
+ *
+ * 1. Using \a LL_PROC_PROTO macro like this:
+ * "int LL_PROC_PROTO(name_of_callback) {"
+ *
+ * 2. Using \a DECLARE_PROC_HANDLER. In this case the consumer should
+ * firstly implement a callback with name prepent by "__" like this:
+ * "static int __name_of_callback(void *data, int write, loff_t pos,
+ * void *buffer, int nob) { ... implementation ...}". Then, this macro
+ * should be used with name of callback (without "__") as its only
+ * argument: "DECLARE_PROC_HANDLER(name_of_callback);"
+ *
+ * The macro \a DECLARE_PROC_HANDLER allows consumer to avoid
+ * duplication of code related to updating \a lenp and \a ppos that
+ * linux kernel passes as arguments to sysctl callback. See
+ * implementation of \a proc_call_handler in linux-proc.c for details.
+ *
+ * @{
+ */
 
 /*
  * Sysctl register
@@ -103,6 +147,24 @@ LL_PROC_PROTO(name)                     
 }
 
 /*
+ * Proc file system APIs
+ */
+typedef read_proc_t                     cfs_read_proc_t;
+typedef write_proc_t                    cfs_write_proc_t;
+typedef struct proc_dir_entry           cfs_proc_dir_entry_t;
+#define cfs_create_proc_entry(n, m, p)  create_proc_entry(n, m, p)
+#define cfs_free_proc_entry(e)          free_proc_entry(e)
+#define cfs_remove_proc_entry(n, e)     remove_proc_entry(n, e)
+
+/** @} libcfs_linux_procfs */
+/** @} libcfs_procfs */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
+
+/*
  * Symbol register
  */
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
@@ -121,18 +183,21 @@ LL_PROC_PROTO(name)                     
 #define cfs_module_put()                module_put(THIS_MODULE)
 #endif
 
-/*
- * Proc file system APIs
- */
-typedef read_proc_t                     cfs_read_proc_t;
-typedef write_proc_t                    cfs_write_proc_t;
-typedef struct proc_dir_entry           cfs_proc_dir_entry_t;
-#define cfs_create_proc_entry(n, m, p)  create_proc_entry(n, m, p)
-#define cfs_free_proc_entry(e)          free_proc_entry(e)
-#define cfs_remove_proc_entry(n, e)     remove_proc_entry(n, e)
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
 
-/*
- * Wait Queue
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_linux_synchronization
+ * @{
+ * \defgroup libcfs_linux_waitqueue Wait queues
+ * wait-queue (linux-specific)
+ *
+ * For linux case, Libcfs implements wait-queue primitives in trivial
+ * way by direct calls of correspondent linux-native primitives.
+ *
+ * \see \ref libcfs_waitqueue for general description.
+ * @{
  */
 #define CFS_TASK_INTERRUPTIBLE          TASK_INTERRUPTIBLE
 #define CFS_TASK_UNINT                  TASK_UNINTERRUPTIBLE
@@ -143,6 +208,14 @@ typedef struct proc_dir_entry           
 typedef wait_queue_t			cfs_waitlink_t;
 typedef wait_queue_head_t		cfs_waitq_t;
 typedef long                            cfs_task_state_t;
+/** @} libcfs_linux_waitqueue */
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
 
 #define cfs_kthread_run(fn, data, fmt, arg...) kthread_run(fn, data, fmt, ##arg)
 
@@ -184,17 +257,47 @@ module_init(init);                      
 module_exit(fini)
 
 /*
- * Signal
+ * interrupt
+ */
+
+#define cfs_in_interrupt() in_interrupt()
+
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
+
+/** \addtogroup libcfs_signal
+ * @{
+ * \defgroup libcfs_linux_signal Signal (linux-specific)
+ * @{
  */
 typedef sigset_t                        cfs_sigset_t;
+/** @} */
+/** @} */
 
-/*
- * Timer
+/** \addtogroup libcfs_timer
+ * @{
+ * \defgroup libcfs_linux_timer Timer (linux-specific)
+ * @{
  */
 typedef struct timer_list cfs_timer_t;
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_schedule
+ * @{
+ * \defgroup libcfs_linux_schedule Schedule (linux-specific)
+ * @{ */
+/** linux-specific infinite timeout */
 #define CFS_MAX_SCHEDULE_TIMEOUT MAX_SCHEDULE_TIMEOUT
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_linux_synchronization
+ * @{
+ * \addtogroup libcfs_linux_waitqueue
+ * @{ */
 #ifndef wait_event_timeout /* Only for RHEL3 2.4.21 kernel */
 #define __wait_event_timeout(wq, condition, timeout, ret)        \
 do {                                                             \
@@ -220,10 +323,7 @@ do {                                    
 		remove_wait_queue(&wq, &__wait);                 \
 	}                                                        \
 } while (0)
-/*
-   retval == 0; condition met; we're good.
-   retval > 0; timed out.
-*/
+
 #define cfs_waitq_wait_event_timeout(wq, condition, timeout, ret)    \
 do {                                                                 \
 	ret = 0;                                                     \
@@ -231,6 +331,29 @@ do {                                    
 		__wait_event_timeout(wq, condition, timeout, ret);   \
 } while (0)
 #else
+/**
+ * Conditional wait for an event with specified timeout. If
+ * \a condition is already satisfied, the call returns
+ * immediately. Otherwise, caller will go to sleep till other thread
+ * issues a signal (e.g. cfs_waitq_signal) or timeout expires.
+ *
+ * See \ref libcfs_waitqueue for background info about wait-queues.
+ *
+ * NOTE: Locking strategy is quite different from the usage of other
+ * wait-queue primitives like cfs_waitq_timedwait. To ensure MT-safety
+ * here, the caller only needs lock protection for check/update of
+ * resource what it waits for. Linux resolves 'lost wakeup' problem
+ * internally, at os scheduler layer.
+ * 
+ * \param wq Wait-queue to be signalled to wake up caller
+ * \param condition Condition to check before sleep. Should return
+ * TRUE if resource is available and the caller needn't sleep.
+ * \param timeout Maximum duration (in machine ticks) of
+ * sleep if no signal happens.
+ * \param ret Local variable of 'int' type to store return code:
+ * - ret == 0; condition met; we're good.
+ * - ret > 0; timed out.
+ */
 #define cfs_waitq_wait_event_timeout(wq, condition, timeout, ret)    \
         ret = wait_event_timeout(wq, condition, timeout)
 #endif
@@ -279,12 +402,22 @@ do {                                    
 						timeout, ret);     \
 } while (0)
 #else
+/**
+ * Conditional interruptible wait for an event with specified timeout.
+ *
+ * Exactly the same as cfs_waitq_wait_event_timeout but ensures sleep
+ * in interruptible way. Upon return, \a ret can also be negative
+ * meaning that sleep was interrupted.
+ */
 #define cfs_waitq_wait_event_interruptible_timeout(wq, c, timeout, ret) \
         ret = wait_event_interruptible_timeout(wq, c, timeout)
 #endif
+/** @} libcfs_linux_waitqueue */
 
-/*
- * atomic
+/**
+ * \defgroup libcfs_linux_atomic Atomics
+ * atomic (use Linux kernel's primitives)
+ * @{
  */
 
 typedef atomic_t cfs_atomic_t;
@@ -297,16 +430,20 @@ typedef atomic_t cfs_atomic_t;
 #define cfs_atomic_add(value, atom)   atomic_add(value, atom)
 #define cfs_atomic_sub(value, atom)   atomic_sub(value, atom)
 
-/*
- * membar
- */
-
-#define cfs_mb() mb()
+/** @} libcfs_linux_atomic */
 
-/*
- * interrupt
+/**
+ * \defgroup libcfs_linux_membar Membar
+ * Memory barrier. Ensures that all CPU-caches are synchronized
+ * avoiding situation when one thread executed 'a=1;' then 'b=2;' but
+ * another thread might observe these changes in reverse order.
+ * @{
  */
+#define cfs_mb() mb()
 
-#define cfs_in_interrupt() in_interrupt()
+/** @} libcfs_linux_membar */
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
 
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/linux/linux-tcpip.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-tcpip.h,v
retrieving revision 1.6
diff -u -p -r1.6 linux-tcpip.h
--- libcfs/include/libcfs/linux/linux-tcpip.h	26 Oct 2009 08:08:25 -0000	1.6
+++ libcfs/include/libcfs/linux/linux-tcpip.h	1 Jan 2010 20:03:33 -0000
@@ -51,6 +51,18 @@
 
 #include <net/sock.h>
 
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_net
+ * @{
+ * \addtogroup libcfs_kernel_net
+ * @{
+ * \defgroup libcfs_linux_net Network API (kernel-space, linux-specific)
+ *
+ * See \ref libcfs_net for generic description.
+ * @{ */
+
+/** This is how linux-kernel libcfs abstracts linux-native socket */
 typedef struct socket   cfs_socket_t;
 
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,72))
@@ -71,7 +83,9 @@ typedef struct socket   cfs_socket_t;
 # define sk_route_caps  route_caps
 #endif
 
+/** obsoleted */
 #define SOCK_SNDBUF(so)         ((so)->sk->sk_sndbuf)
+/** returns TRUE if socket is still in 'ENOMEM' state */
 #define SOCK_TEST_NOSPACE(so)   test_bit(SOCK_NOSPACE, &(so)->flags)
 
 static inline int
@@ -80,6 +94,7 @@ libcfs_sock_error(struct socket *sock)
         return sock->sk->sk_err;
 }
 
+/** returns # not-ACK-ed bytes in send buffer */
 static inline int
 libcfs_sock_wmem_queued(struct socket *sock)
 {
@@ -93,4 +108,8 @@ libcfs_sock_wmem_queued(struct socket *s
 #define DEFAULT_NET	(NULL)
 #endif
 
+/** @} libcfs_linux_net */
+/** @} libcfs_kernel_net */
+/** @} libcfs_net */
+/** @} libcfs */
 #endif
Index: libcfs/include/libcfs/linux/linux-time.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/linux-time.h,v
retrieving revision 1.5
diff -u -p -r1.5 linux-time.h
--- libcfs/include/libcfs/linux/linux-time.h	10 Feb 2009 18:33:32 -0000	1.5
+++ libcfs/include/libcfs/linux/linux-time.h	1 Jan 2010 20:03:33 -0000
@@ -51,9 +51,16 @@
 #error This include is only for kernel use.
 #endif
 
-/* Portable time API */
+/** \addtogroup libcfs
+ * @{ */
 
-/*
+/** \addtogroup libcfs_time
+ * @{ */
+
+/**
+ * \defgroup libcfs_linux_time Time (linux-specific)
+ * Portable time API for linux platform
+ *
  * Platform provides three opaque data-types:
  *
  *  cfs_time_t        represents point in time. This is internal kernel
@@ -87,7 +94,7 @@
  *
  *  CFS_TIME_FORMAT
  *  CFS_DURATION_FORMAT
- *
+ * @{
  */
 
 #define ONE_BILLION ((u_int64_t)1000000000)
@@ -110,7 +117,7 @@
 
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 
-/*
+/**
  * old kernels---CURRENT_TIME is struct timeval
  */
 typedef struct timeval cfs_fs_time_t;
@@ -126,7 +133,7 @@ static inline void cfs_fs_time_nsec(cfs_
         s->tv_nsec = t->tv_usec * 1000;
 }
 
-/*
+/**
  * internal helper function used by cfs_fs_time_before*()
  */
 static inline unsigned long long __cfs_fs_time_flat(cfs_fs_time_t *t)
@@ -158,7 +165,7 @@ static inline void cfs_fs_time_nsec(cfs_
         *s = *t;
 }
 
-/*
+/**
  * internal helper function used by cfs_fs_time_before*()
  */
 static inline unsigned long long __cfs_fs_time_flat(cfs_fs_time_t *t)
@@ -175,7 +182,9 @@ static inline unsigned long long __cfs_f
  * Generic kernel stuff
  */
 
-typedef unsigned long cfs_time_t;      /* jiffies */
+/** jiffies */
+typedef unsigned long cfs_time_t;
+/** jiffies at one point in time minus jiffies at another */
 typedef long cfs_duration_t;
 
 
@@ -298,7 +307,7 @@ static inline int cfs_time_beforeq_64(__
 #define cfs_time_beforeq_64 cfs_time_beforeq
 #endif
 
-/*
+/**
  * One jiffy
  */
 #define CFS_TICK                (1)
@@ -308,6 +317,10 @@ static inline int cfs_time_beforeq_64(__
 
 #define cfs_do_gettimeofday(tv) do_gettimeofday(tv)
 
+/* @} libcfs_linux_time */
+/* @} libcfs_time */
+/* @} libcfs */
+
 #endif /* __LIBCFS_LINUX_LINUX_TIME_H__ */
 /*
  * Local variables:
Index: libcfs/include/libcfs/linux/portals_compat25.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/linux/portals_compat25.h,v
retrieving revision 1.7
diff -u -p -r1.7 portals_compat25.h
--- libcfs/include/libcfs/linux/portals_compat25.h	1 Dec 2009 02:23:48 -0000	1.7
+++ libcfs/include/libcfs/linux/portals_compat25.h	1 Jan 2010 20:03:33 -0000
@@ -37,6 +37,9 @@
 #ifndef __LIBCFS_LINUX_PORTALS_COMPAT_H__
 #define __LIBCFS_LINUX_PORTALS_COMPAT_H__
 
+/** \addtogroup libcfs
+ * @{ */
+
 // XXX BUG 1511 -- remove this stanza and all callers when bug 1511 is resolved
 #if defined(SPINLOCK_DEBUG) && SPINLOCK_DEBUG
 # if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)) || defined(CONFIG_RH_2_4_20)
@@ -96,11 +99,36 @@
 
 #endif
 
-# define cfs_wait_event_interruptible(wq, condition, ret)               \
+/** \addtogroup libcfs_synchronization
+ * @{
+ * \addtogroup libcfs_linux_synchronization
+ * @{
+ * \addtogroup libcfs_linux_waitqueue
+ * @{ */
+
+/**
+ * The same as cfs_waitq_wait_event_interruptible_timeout, but
+ * without timeout: the caller will sleep till signal.
+ */
+# define cfs_wait_event_interruptible(wq, condition, ret)       \
          ret = wait_event_interruptible(wq, condition)
+/**
+ * The same as cfs_wait_event_interruptible, but with exclusive semantic:
+ * only one waiter in given wait-queue will be woken up by
+ * cfs_waitq_signal.
+ */
 # define cfs_wait_event_interruptible_exclusive(wq, condition, ret)     \
          ret = wait_event_interruptible(wq, condition)
 
+/** @} libcfs_linux_waitqueue */
+/** @} libcfs_linux_synchronization */
+/** @} libcfs_synchronization */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_linux_misc
+ * @{ */
+
 #if defined(__arch_um__) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,4,20))
 #define UML_PID(tsk) ((tsk)->thread.extern_pid)
 #elif defined(__arch_um__) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
@@ -144,6 +172,14 @@ typedef unsigned long cpumask_t;
 #define __fls fls
 #endif
 
+/** @} libcfs_linux_misc */
+/** @} libcfs_misc */
+
+/** \addtogroup libcfs_procfs
+ * @{
+ * \addtogroup libcfs_linux_procfs
+ * @{ */
+
 #define ll_proc_dointvec(table, write, filp, buffer, lenp, ppos)        \
         proc_dointvec(table, write, filp, buffer, lenp, ppos);
 
@@ -162,4 +198,8 @@ int proc_call_handler(void *data, int wr
                       int (*handler)(void *data, int write,
                                      loff_t pos, void *buffer, int len));
 
+/** @} libcfs_linux_procfs */
+/** @} libcfs_procfs */
+
+/** @} libcfs */
 #endif /* _PORTALS_COMPAT_H */
Index: libcfs/include/libcfs/posix/libcfs.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/posix/libcfs.h,v
retrieving revision 1.13
diff -u -p -r1.13 libcfs.h
--- libcfs/include/libcfs/posix/libcfs.h	9 Oct 2009 13:46:20 -0000	1.13
+++ libcfs/include/libcfs/posix/libcfs.h	1 Jan 2010 20:03:33 -0000
@@ -106,13 +106,35 @@
 #include <libcfs/user-bitops.h>
 #include <libcfs/posix/posix-kernelcomm.h>
 
+/** \addtogroup libcfs
+ * @{ */
+
+/** \addtogroup libcfs_time
+ * @{
+ * \addtogroup libcfs_user_time
+ * @{ */
 # define do_gettimeofday(tv) gettimeofday(tv, NULL);
 typedef unsigned long long cycles_t;
+/** @} */
+/** @} */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_userspace_misc
+ * @{ */
 
 #define IS_ERR(a) ((unsigned long)(a) > (unsigned long)-1000L)
 #define PTR_ERR(a) ((long)(a))
 #define ERR_PTR(a) ((void*)((long)(a)))
 
+/* XXX Maxim: We shouldn't duplicate defining LUSTRE_LNET_PID in all
+ * platform-specific libcfs.h headers. Move it to common libcfs.h. */
+/** initial pid  */
+#define LUSTRE_LNET_PID          12345
+
+/** @} */
+/** @} */
+
 /* this goes in posix-fs.h */
 #include <sys/mount.h>
 
@@ -120,14 +142,33 @@ typedef unsigned long long cycles_t;
 #include <mntent.h>
 #endif
 
+/**
+ * \addtogroup libcfs_fs
+ * @{
+ * \defgroup libcfs_user_fs File operations for user-space users
+ * A few file-related bits for user-space
+ *
+ * Libcfs provides pretty limited file-related functionality for
+ * user-space:
+ * - types cfs_file_t, cfs_dentry_t and cfs_dirent_t
+ * - CFS_IFTODT and CFS_DTTOIF - conversion macros for directory type
+ * - read_actor_t - callback type for "sendfile" file operation
+ * @{
+ */
 typedef struct file cfs_file_t;
 typedef struct dentry cfs_dentry_t;
 #ifdef __linux__
 typedef struct dirent64 cfs_dirent_t;
 #endif
+/* @} libcfs_user_fs */
+/* @} libcfs_fs */
 
+/** \addtogroup libcfs_types
+ * @{
+ * \addtogroup libcfs_user_types
+ * @{ */
 #ifdef __linux__
-/* Userpace byte flipping */
+/* User-space byte flipping */
 # include <endian.h>
 # include <byteswap.h>
 # define __swab16(x) bswap_16(x)
@@ -194,7 +235,20 @@ typedef struct dirent64 cfs_dirent_t;
 #define __swab32s(x)                            do { *(x) = __swab32(*(x)); } while (0)
 #define __swab64s(x)                            do { *(x) = __swab64(*(x)); } while (0)
 #endif
+/** @} libcfs_user_types */
+/** @} libcfs_types */
 
+/** \addtogroup libcfs_debug
+ * @{
+ * \defgroup libcfs_user_debug Debugging (user-space)
+ * User-space implementation of Libcfs debug primitives.
+ *
+ * On some platforms (e.g. i386) the stack is very limited resource
+ * for kernel threads. This is not the case for user-space and Libcfs
+ * provides only stubs for controlling depth of stack in user-space.
+ *
+ * @{
+ */
 
 # ifndef THREAD_SIZE /* x86_64 linux has THREAD_SIZE in userspace */
 #  define THREAD_SIZE 8192
@@ -205,22 +259,36 @@ typedef struct dirent64 cfs_dirent_t;
 #define CHECK_STACK() do { } while(0)
 #define CDEBUG_STACK() (0L)
 
-/* initial pid  */
-#define LUSTRE_LNET_PID          12345
-
 #define ENTRY_NESTING_SUPPORT (1)
 #define ENTRY_NESTING   do {;} while (0)
 #define EXIT_NESTING   do {;} while (0)
 #define __current_nesting_level() (0)
+/** @} libcfs_user_debug */
+/** @} libcfs_debug */
 
-/**
- * Platform specific declarations for cfs_curproc API (libcfs/curproc.h)
- *
- * Implementation is in linux-curproc.c
+/** \addtogroup libcfs_curproc
+ * @{
+ * \defgroup libcfs_userspace_curproc Curproc (user-space)
+ * User-space implementation of curproc API (libcfs/curproc.h)
+ * @{
  */
+
+/** Maximum length of process name */
 #define CFS_CURPROC_COMM_MAX (sizeof ((struct task_struct *)0)->comm)
 
+/** obsoleted */
 typedef __u32 cfs_kernel_cap_t;
+/* XXX Maxim: It's useless to define cfs_kernel_cap_t in posix part of
+ * libcfs. The only user of cfs_kernel_cap_t is linux-specific lvfs
+ * that has nothing to do with posix */
+
+/** @} libcfs_userspace_curproc */
+/** @} libcfs_curproc */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_userspace_misc
+ * @{ */
 
 /**
  * Module support (probably shouldn't be used in generic code?)
@@ -410,6 +478,14 @@ static inline void radix_tree_preload_en
 {
 }
 
+/** @} libcfs_userspace_misc */
+/** @} libcfs_misc */
+
+/** \addtogroup libcfs_fs
+ * @{
+ * \addtogroup libcfs_user_fs
+ * @{ */
+
 typedef ssize_t (*read_actor_t)();
 
 #define CFS_IFSHIFT 12
@@ -417,4 +493,8 @@ typedef ssize_t (*read_actor_t)();
 #define CFS_IFTODT(type)           (((type) & S_IFMT) >> CFS_IFSHIFT)
 #define CFS_DTTOIF(dirtype)        ((dirtype) << CFS_IFSHIFT)
 
+/** @} libcfs_user_fs */
+/** @} libcfs_fs */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/posix/posix-kernelcomm.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/posix/posix-kernelcomm.h,v
retrieving revision 1.1
diff -u -p -r1.1 posix-kernelcomm.h
--- libcfs/include/libcfs/posix/posix-kernelcomm.h	28 Jul 2009 19:42:54 -0000	1.1
+++ libcfs/include/libcfs/posix/posix-kernelcomm.h	1 Jan 2010 20:03:33 -0000
@@ -47,6 +47,14 @@
 #error Do not #include this file directly. #include <libcfs/libcfs.h> instead
 #endif
 
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_lnl
+ * @{
+ * \defgroup libcfs_userspace_lnl Kernel <-> user-space communication (user-space)
+ * User-space side of kernel <-> user-space communication subsystem.
+ * @{ */
+
 typedef int lustre_netlink;
 int libcfs_ulnl_start(lustre_netlink *l, int groups);
 int libcfs_ulnl_stop(lustre_netlink *l);
@@ -55,5 +63,8 @@ int libcfs_ulnl_msg_get(lustre_netlink *
                         struct lnl_hdr **lnlhh);
 int libcfs_ulnl_msg_free(struct lnl_hdr **lnlhh);
 
+/** @} libcfs_userspace_lnl */
+/** @} libcfs_lnl */
+/** @} libcfs */
 #endif
 
Index: libcfs/include/libcfs/posix/posix-types.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/posix/posix-types.h,v
retrieving revision 1.6
diff -u -p -r1.6 posix-types.h
--- libcfs/include/libcfs/posix/posix-types.h	5 Aug 2009 11:37:51 -0000	1.6
+++ libcfs/include/libcfs/posix/posix-types.h	1 Jan 2010 20:03:33 -0000
@@ -43,6 +43,24 @@
 #define _LUSTRE_POSIX_TYPES_H
 
 #include <asm/types.h>
+
+/**
+ * \addtogroup libcfs
+ * @{
+ *
+ * \addtogroup libcfs_types
+ * @{
+ *
+ *\defgroup libcfs_user_types Generic types (user-space)
+ *
+ * Definitions of some generic types and swab primitives for user-space.
+ *
+ * For user-space, Libcfs adds types __s8, __u8, __s16, __u16, __s32,
+ * __u32, __s64, __u64, ulong_ptr_t, long_ptr_t and a bunch of swab
+ * primitives to be found in posix/libcfs.h
+ * @{
+ */
+
 #ifndef HAVE_UMODE_T
 typedef unsigned short umode_t;
 #endif
@@ -72,4 +90,8 @@ typedef unsigned long long __u64;
 typedef unsigned long ulong_ptr_t;
 typedef long long_ptr_t;
 
+/** @} libcfs_user_types */
+/** @} libcfs_types */
+/** @} libcfs */
+
 #endif
Index: libcfs/include/libcfs/posix/posix-wordsize.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/include/libcfs/posix/posix-wordsize.h,v
retrieving revision 1.9
diff -u -p -r1.9 posix-wordsize.h
--- libcfs/include/libcfs/posix/posix-wordsize.h	26 Aug 2009 09:41:26 -0000	1.9
+++ libcfs/include/libcfs/posix/posix-wordsize.h	1 Jan 2010 20:03:33 -0000
@@ -55,6 +55,14 @@
 # include "ioctl.h"
 #endif
 
+/** \addtogroup libcfs
+ * @{ */
+
+/** \addtogroup libcfs_misc
+ * @{
+ * \defgroup libcfs_userspace_misc Misc. (user-space)
+ * @{ */
+
 # define CFS_MODULE_PARM(name, t, type, perm, desc)
 #define PORTAL_SYMBOL_GET(x) inter_module_get(#x)
 #define PORTAL_SYMBOL_PUT(x) inter_module_put(#x)
@@ -72,10 +80,17 @@
 #  define BITS_PER_LONG __WORDSIZE
 #endif
 
+/** @} libcfs_userspace_misc */
+/** @} libcfs_misc */
 
-/******************************************************************************/
-/* Light-weight trace
- * Support for temporary event tracing with minimal Heisenberg effect. */
+/**
+ * \addtogroup libcfs_lwt
+ * @{
+ * \defgroup libcfs_user_lwt Light-weight tracing (user-space)
+ *
+ * LWT is not implemented for user-space yet.
+ * @{
+ */
 #define LWT_SUPPORT  0
 
 #define LWT_MEMORY   (16<<20)
@@ -96,11 +111,23 @@ typedef struct {
 #if LWT_SUPPORT
 #define LWT_EVENT(p1,p2,p3,p4)     /* no userland implementation yet */
 #endif /* LWT_SUPPORT */
+/** @} libcfs_lwt */
+/** @} libcfs_user_lwt */
 
 /* ------------------------------------------------------------------ */
 
+/** \addtogroup libcfs_ioctl
+ * @{
+ * \defgroup libcfs_userspace_ioctl Pseudo devices and ioctls (user-space)
+ * @{ */
 #define IOCTL_LIBCFS_TYPE long
+/** @} */
+/** @} */
 
+/** \addtogroup libcfs_misc
+ * @{
+ * \addtogroup libcfs_userspace_misc
+ * @{ */
 
 #if BITS_PER_LONG > 32
 # define LI_POISON ((int)0x5a5a5a5a5a5a5a5a)
@@ -159,4 +186,9 @@ typedef struct {
  */
 # define LPPID "%d"
 
+/** @} libcfs_userspace_misc */
+/** @} libcfs_misc */
+
+/** @} libcfs */
+
 #endif
Index: libcfs/libcfs/debug.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/debug.c,v
retrieving revision 1.19
diff -u -p -r1.19 debug.c
--- libcfs/libcfs/debug.c	6 Nov 2009 13:54:48 -0000	1.19
+++ libcfs/libcfs/debug.c	1 Jan 2010 20:03:33 -0000
@@ -110,6 +110,7 @@ CFS_MODULE_PARM(libcfs_panic_on_lbug, "i
                 "Lustre kernel panic on LBUG");
 EXPORT_SYMBOL(libcfs_panic_on_lbug);
 
+/** # bytes "in use" (allocated but not freed yet) */
 atomic_t libcfs_kmemory = ATOMIC_INIT(0);
 EXPORT_SYMBOL(libcfs_kmemory);
 
Index: libcfs/libcfs/hash.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/hash.c,v
retrieving revision 1.1
diff -u -p -r1.1 hash.c
--- libcfs/libcfs/hash.c	1 Dec 2009 02:23:59 -0000	1.1
+++ libcfs/libcfs/hash.c	1 Jan 2010 20:03:33 -0000
@@ -86,13 +86,13 @@ cfs_hash_wunlock(cfs_hash_t *hs)
 }
 
 /**
- * Initialize new libcfs hash, where:
- * @name     - Descriptive hash name
- * @cur_bits - Initial hash table size, in bits
- * @max_bits - Maximum allowed hash table resize, in bits
- * @ops      - Registered hash table operations
- * @flags    - CFS_HASH_REHASH enable synamic hash resizing
- *           - CFS_HASH_SORT enable chained hash sort
+ * Initialize new libcfs hash.
+ * \param name     Descriptive hash name
+ * \param cur_bits Initial hash table size, in bits
+ * \param max_bits Maximum allowed hash table resize, in bits
+ * \param ops      Registered hash table operations
+ * \param flags    CFS_HASH_REHASH enables dynamic hash resizing;
+ *                 CFS_HASH_SORT enables chained hash sort
  */
 cfs_hash_t *
 cfs_hash_create(char *name, unsigned int cur_bits,
@@ -157,7 +157,7 @@ cfs_hash_create(char *name, unsigned int
 CFS_EXPORT_SYMBOL(cfs_hash_create);
 
 /**
- * Cleanup libcfs hash @hs.
+ * Cleanup libcfs hash \a hs.
  */
 void
 cfs_hash_destroy(cfs_hash_t *hs)
@@ -219,7 +219,7 @@ cfs_hash_rehash_bits(cfs_hash_t *hs)
 }
 
 /**
- * Add item @hnode to libcfs hash @hs using @key.  The registered
+ * Add item \a hnode to libcfs hash \a hs using \a key.  The registered
  * ops->hs_get function will be called when the item is added.
  */
 void
@@ -287,9 +287,9 @@ cfs_hash_findadd_unique_hnode(cfs_hash_t
 }
 
 /**
- * Add item @hnode to libcfs hash @hs using @key.  The registered
+ * Add item \a hnode to libcfs hash \a hs using \a key.  The registered
  * ops->hs_get function will be called if the item was added.
- * Returns 0 on success or -EALREADY on key collisions.
+ * \return 0 on success or -EALREADY on key collisions.
  */
 int
 cfs_hash_add_unique(cfs_hash_t *hs, void *key, struct hlist_node *hnode)
@@ -307,7 +307,7 @@ cfs_hash_add_unique(cfs_hash_t *hs, void
 CFS_EXPORT_SYMBOL(cfs_hash_add_unique);
 
 /**
- * Add item @hnode to libcfs hash @hs using @key.  If this @key
+ * Add item \a hnode to libcfs hash \a hs using \a key.  If this \a key
  * already exists in the hash then ops->hs_get will be called on the
  * conflicting entry and that entry will be returned to the caller.
  * Otherwise ops->hs_get is called on the item which was added.
@@ -328,7 +328,7 @@ cfs_hash_findadd_unique(cfs_hash_t *hs, 
 CFS_EXPORT_SYMBOL(cfs_hash_findadd_unique);
 
 /**
- * Delete item @hnode from the libcfs hash @hs using @key.  The @key
+ * Delete item \a hnode from the libcfs hash \a hs using \a key. The \a key
  * is required to ensure the correct hash bucket is locked since there
  * is no direct linkage from the item to the bucket.  The object
  * removed from the hash will be returned and obs->hs_put is called
@@ -360,9 +360,9 @@ cfs_hash_del(cfs_hash_t *hs, void *key, 
 CFS_EXPORT_SYMBOL(cfs_hash_del);
 
 /**
- * Delete item given @key in libcfs hash @hs.  The first @key found in
+ * Delete item given \a key in libcfs hash \a hs.  The first \a key found in
  * the hash will be removed, if the key exists multiple times in the hash
- * @hs this function must be called once per key.  The removed object
+ * \a hs this function must be called once per key.  The removed object
  * will be returned and ops->hs_put is called on the removed object.
  */
 void *
@@ -392,12 +392,12 @@ cfs_hash_del_key(cfs_hash_t *hs, void *k
 CFS_EXPORT_SYMBOL(cfs_hash_del_key);
 
 /**
- * Lookup an item using @key in the libcfs hash @hs and return it.
- * If the @key is found in the hash hs->hs_get() is called and the
+ * Lookup an item using \a key in the libcfs hash \a hs and return it.
+ * If the \a key is found in the hash hs->hs_get() is called and the
  * matching objects is returned.  It is the callers responsibility
  * to call the counterpart ops->hs_put using the cfs_hash_put() macro
- * when when finished with the object.  If the @key was not found
- * in the hash @hs NULL is returned.
+ * when when finished with the object.  If the \a key was not found
+ * in the hash \a hs NULL is returned.
  */
 void *
 cfs_hash_lookup(cfs_hash_t *hs, void *key)
@@ -426,8 +426,8 @@ cfs_hash_lookup(cfs_hash_t *hs, void *ke
 CFS_EXPORT_SYMBOL(cfs_hash_lookup);
 
 /**
- * For each item in the libcfs hash @hs call the passed callback @func
- * and pass to it as an argument each hash item and the private @data.
+ * For each item in the libcfs hash \a hs call the passed callback \a func
+ * and pass to it as an argument each hash item and the private \a data.
  * Before each callback ops->hs_get will be called, and after each
  * callback ops->hs_put will be called.  Finally, during the callback
  * the bucket lock is held so the callback must never sleep.
@@ -460,8 +460,8 @@ cfs_hash_for_each(cfs_hash_t *hs,
 CFS_EXPORT_SYMBOL(cfs_hash_for_each);
 
 /**
- * For each item in the libcfs hash @hs call the passed callback @func
- * and pass to it as an argument each hash item and the private @data.
+ * For each item in the libcfs hash \a hs call the passed callback \a func
+ * and pass to it as an argument each hash item and the private \a data.
  * Before each callback ops->hs_get will be called, and after each
  * callback ops->hs_put will be called.  During the callback the
  * bucket lock will not be held will allows for the current item
@@ -499,11 +499,11 @@ cfs_hash_for_each_safe(cfs_hash_t *hs,
 CFS_EXPORT_SYMBOL(cfs_hash_for_each_safe);
 
 /**
- * For each hash bucket in the libcfs hash @hs call the passed callback
- * @func until all the hash buckets are empty.  The passed callback @func
+ * For each hash bucket in the libcfs hash \a hs call the passed callback
+ * \a func until all the hash buckets are empty.  The passed callback \a func
  * or the previously registered callback hs->hs_put must remove the item
  * from the hash.  You may either use the cfs_hash_del() or hlist_del()
- * functions.  No rwlocks will be held during the callback @func it is
+ * functions.  No rwlocks will be held during the callback \a func it is
  * safe to sleep if needed.  This function will not terminate until the
  * hash is empty.  Note it is still possible to concurrently add new
  * items in to the hash.  It is the callers responsibility to ensure
@@ -540,10 +540,10 @@ restart:
 }
 CFS_EXPORT_SYMBOL(cfs_hash_for_each_empty);
 
-/*
- * For each item in the libcfs hash @hs which matches the @key call
- * the passed callback @func and pass to it as an argument each hash
- * item and the private @data.  Before each callback ops->hs_get will
+/**
+ * For each item in the libcfs hash \a hs which matches the \a key call
+ * the passed callback \a func and pass to it as an argument each hash
+ * item and the private \a data.  Before each callback ops->hs_get will
  * be called, and after each callback ops->hs_put will be called.
  * Finally, during the callback the bucket lock is held so the
  * callback must never sleep.
@@ -581,15 +581,15 @@ cfs_hash_for_each_key(cfs_hash_t *hs, vo
 CFS_EXPORT_SYMBOL(cfs_hash_for_each_key);
 
 /**
- * Rehash the libcfs hash @hs to the given @bits.  This can be used
+ * Rehash the libcfs hash \a hs to the given \a bits.  This can be used
  * to grow the hash size when excessive chaining is detected, or to
  * shrink the hash when it is larger than needed.  When the CFS_HASH_REHASH
- * flag is set in @hs the libcfs hash may be dynamically rehashed
+ * flag is set in \a hs the libcfs hash may be dynamically rehashed
  * during addition or removal if the hash's theta value exceeds
  * either the hs->hs_min_theta or hs->max_theta values.  By default
  * these values are tuned to keep the chained hash depth small, and
  * this approach assumes a reasonably uniform hashing function.  The
- * theta thresholds for @hs are tunable via cfs_hash_set_theta().
+ * theta thresholds for \a hs are tunable via cfs_hash_set_theta().
  */
 int
 cfs_hash_rehash(cfs_hash_t *hs, int bits)
@@ -696,9 +696,9 @@ cfs_hash_rehash(cfs_hash_t *hs, int bits
 CFS_EXPORT_SYMBOL(cfs_hash_rehash);
 
 /**
- * Rehash the object referenced by @hnode in the libcfs hash @hs.  The
- * @old_key must be provided to locate the objects previous location
- * in the hash, and the @new_key will be used to reinsert the object.
+ * Rehash the object referenced by \a hnode in the libcfs hash \a hs.  The
+ * \a old_key must be provided to locate the objects previous location
+ * in the hash, and the \a new_key will be used to reinsert the object.
  * Use this function instead of a cfs_hash_add() + cfs_hash_del()
  * combo when it is critical that there is no window in time where the
  * object is missing from the hash.  When an object is being rehashed
Index: libcfs/libcfs/libcfs_string.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/libcfs_string.c,v
retrieving revision 1.2
diff -u -p -r1.2 libcfs_string.c
--- libcfs/libcfs/libcfs_string.c	13 Mar 2009 08:17:27 -0000	1.2
+++ libcfs/libcfs/libcfs_string.c	1 Jan 2010 20:03:33 -0000
@@ -65,7 +65,38 @@ static int libcfs_strncasecmp(const char
         return tolower(*(unsigned char *)s1) - tolower(*(unsigned char *)s2);
 }
 
-/* Convert a text string to a bitmask */
+/**
+ * Convert a text string to a bitmask.
+ *
+ * The function initially sets \a newmask to \a minmask (starting
+ * point), then iterates over \a str looking for tokens or operators
+ * ('+' or '-') and updates \a newmask accordingly. When the whole \a
+ * str is parsed, \a newmask is stored in \a *oldmask. See inline
+ * comments inside implementation for details.
+ *
+ * The token is an arbitrary sub-string that doesn't contain
+ * white-spaces. Usually, the token is some descriptive word
+ * corresponding a bit. \a bit2str realizes this correspondence: \a
+ * bit2str(i) should return a string corresponding to the bit \a i.
+ *
+ * The word "ALL" is always valid token. When appears, forces
+ * \a newmask to be reinitialized (i.e. all previous tokens are
+ * effectively ignored):
+ * - to \a minmask if "ALL" is in scope of '-' operator
+ * - to \a allmask otherwise.
+ *
+ * \param str a list of tokens separated by whitespace
+ * \param bit2str a callback converting given bit to its string representation
+ * \param oldmask serves two-fold purpose:
+ * - as starting point if an operator appears in \a str prior any tokens
+ * - as storage for output of the function (\a newmask calculated)
+ * \param minmask starting point if no operators appear in \a str
+ * prior any tokens
+ * \param allmask mask corresponding to magic word 'ALL' in \a str
+ * \return 0 if OK, -EINVAL if error (\a str is misformatted)
+ *
+ * NOTE: tokens are case-insensitive.
+ */
 int libcfs_str2mask(const char *str, const char *(*bit2str)(int bit),
                     int *oldmask, int minmask, int allmask)
 {
@@ -129,7 +160,15 @@ int libcfs_str2mask(const char *str, con
         return 0;
 }
 
-/* Duplicate a string in a platform-independent way */
+/**
+ * Duplicate a string in a platform-independent way.
+ *
+ * \param str string to duplicate
+ * \param flags flags for libcfs memory allocator
+ * \return pointer to duplicated string or NULL if allocation failed
+ *
+ * NOTE: Must free with cfs_free().
+ */
 char *libcfs_strdup(const char *str, u_int32_t flags)
 {
         size_t lenz; /* length of str + zero byte */
Index: libcfs/libcfs/tracefile.h
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/tracefile.h,v
retrieving revision 1.7
diff -u -p -r1.7 tracefile.h
--- libcfs/libcfs/tracefile.h	30 May 2009 14:51:50 -0000	1.7
+++ libcfs/libcfs/tracefile.h	1 Jan 2010 20:03:35 -0000
@@ -47,6 +47,58 @@
 #error Unsupported operating system.
 #endif
 
+/** \addtogroup libcfs
+ * @{
+ * \addtogroup libcfs_debug
+ * @{ */
+
+/**
+ * \defgroup libcfs_tracefile Tracefile subsystem
+ * Tracefile types and primitives
+ *
+ * Tracefile subsystem of Libcfs is a facility for in-kernel users
+ * allowing light-weight gathering log messages and dumping them to a
+ * file when requested.
+ *
+ * The core idea is to print debug messages in memory, into pre-allocated
+ * cpu-bound sets of kernel pages and initiate much more expensive
+ * dumping only when requested.
+ *
+ * Tracefile subsystem maintains global double-indexed array \a trace_data.
+ * The first index is so-called TCD type where "TCD" stands for "Trace CPU
+ * Data". This index reflects whether debug print happened in interrupt
+ * context or not. Such indexing is needed to handle nested prints
+ * properly. The second index is CPU number to avoid extra locking:
+ * for any given CPU number and interrupt level only one debug print
+ * may progress at any point in time. 
+ *
+ * The elements of \a trace_data array are essentially of \a trace_cpu_data
+ * type. This type nests the list of TCD pages (\a tcd_pages) of
+ * struct \a trace_page type. \a trace_page is a wrapper around kernel
+ * page adding some tracefile-specific info like how many bytes are
+ * already printed to this page.
+ *
+ * When a debug print is requested, tracefile subsystem just picks up
+ * proper element from \a trace_data array (based on current interrupt
+ * level and CPU number where the print occurred), gets a "tage" (TCD
+ * page) from the list (or allocates new tage if necessary) and
+ * prints out debug message into kernel page associated with the tage.
+ *
+ * To dump gathered logs on disk, tracefile subsystem firstly collects
+ * all tages into single list, then iterates over this list and writes
+ * the content of kernel pages associated with tages into file.
+ *
+ * The API that tracefile subsystem provides for the rest of Libcfs
+ * consist of four functions:
+ * - libcfs_debug_vmsg2 - to print debug message into tracefile buffers
+ * - tracefile_dump_all_pages - to save all gathered logs on disk
+ * - trace_flush_pages - to clear tracefile buffers (discard all gathered logs)
+ * - libcfs_assertion_failed - a helper to initiate LBUG (see
+ * \ref libcfs_assert for details)
+ *
+ * @{
+ */
+
 /* trace file lock routines */
 
 #define TRACEFILE_NAME_SIZE 1024
@@ -106,13 +158,14 @@ extern int  trace_max_debug_mb(void);
 
 #define TRACEFILE_SIZE (500 << 20)
 
-/* Size of a buffer for sprinting console messages if we can't get a page
+/** Size of a buffer for sprinting console messages if we can't get a page
  * from system */
 #define TRACE_CONSOLE_BUFFER_SIZE   1024
 
+/** Per-CPU trace data */
 union trace_data_union {
 	struct trace_cpu_data {
-		/*
+		/**
 		 * Even though this structure is meant to be per-CPU, locking
 		 * is needed because in some places the data may be accessed
 		 * from other CPUs. This lock is directly used in trace_get_tcd
@@ -122,14 +175,14 @@ union trace_data_union {
 		spinlock_t              tcd_lock;
 		unsigned long           tcd_lock_flags;
 
-		/*
+		/**
 		 * pages with trace records not yet processed by tracefiled.
 		 */
 		struct list_head        tcd_pages;
-		/* number of pages on ->tcd_pages */
+		/** number of pages on ->tcd_pages */
 		unsigned long           tcd_cur_pages;
 
-		/*
+		/**
 		 * pages with trace records already processed by
 		 * tracefiled. These pages are kept in memory, so that some
 		 * portion of log can be written in the event of LBUG. This
@@ -140,10 +193,10 @@ union trace_data_union {
 		 * discarded when list grows too large.
 		 */
 		struct list_head        tcd_daemon_pages;
-		/* number of pages on ->tcd_daemon_pages */
+		/** number of pages on ->tcd_daemon_pages */
 		unsigned long           tcd_cur_daemon_pages;
 
-		/*
+		/**
 		 * Maximal number of pages allowed on ->tcd_pages and
 		 * ->tcd_daemon_pages each.
 		 * Always TCD_MAX_PAGES * tcd_pages_factor / 100 in current
@@ -151,7 +204,7 @@ union trace_data_union {
 		 */
 		unsigned long           tcd_max_pages;
 
-		/*
+		/**
 		 * preallocated pages to write trace records into. Pages from
 		 * ->tcd_stock_pages are moved to ->tcd_pages by
 		 * portals_debug_msg().
@@ -174,13 +227,13 @@ union trace_data_union {
 		 * emitted in non-blocking contexts. Which is quite unlikely.
 		 */
 		struct list_head        tcd_stock_pages;
-		/* number of pages on ->tcd_stock_pages */
+		/** number of pages on ->tcd_stock_pages */
 		unsigned long           tcd_cur_stock_pages;
 
 		unsigned short          tcd_shutting_down;
 		unsigned short          tcd_cpu;
 		unsigned short          tcd_type;
-		/* The factors to share debug memory. */
+		/** The factors to share debug memory. */
 		unsigned short          tcd_pages_factor;
 	} tcd;
 	char __pad[L1_CACHE_ALIGN(sizeof(struct trace_cpu_data))];
@@ -201,9 +254,10 @@ extern union trace_data_union (*trace_da
 
 /* XXX nikita: this declaration is internal to tracefile.c and should probably
  * be moved there */
+/** collection of tages (trace data pages) */
 struct page_collection {
 	struct list_head        pc_pages;
-	/*
+	/**
 	 * spin-lock protecting ->pc_pages. It is taken by smp_call_function()
 	 * call-back functions. XXX nikita: Which is horrible: all processors
 	 * receive NMI at the same time only to be serialized by this
@@ -211,7 +265,7 @@ struct page_collection {
 	 * NR_CPUS elements accessed locklessly.
 	 */
 	spinlock_t              pc_lock;
-	/*
+	/**
 	 * if this flag is set, collect_pages() will spill both
 	 * ->tcd_daemon_pages and ->tcd_pages to the ->pc_pages. Otherwise,
 	 * only ->tcd_pages are spilled.
@@ -229,30 +283,30 @@ struct tracefiled_ctl {
 	atomic_t                tctl_shutdown;
 };
 
-/*
+/**
  * small data-structure for each page owned by tracefiled.
  */
 /* XXX nikita: this declaration is internal to tracefile.c and should probably
  * be moved there */
 struct trace_page {
-	/*
+	/**
 	 * page itself
 	 */
 	cfs_page_t      *page;
-	/*
+	/**
 	 * linkage into one of the lists in trace_data_union or
 	 * page_collection
 	 */
 	struct list_head linkage;
-	/*
+	/**
 	 * number of bytes used within this page
 	 */
 	unsigned int     used;
-	/*
+	/**
 	 * cpu that owns this page
 	 */
 	unsigned short   cpu;
-	/*
+	/**
 	 * type(context) of this page
 	 */
 	unsigned short   type;
@@ -319,7 +373,7 @@ int tcd_owns_tage(struct trace_cpu_data 
 extern void trace_assertion_failed(const char *str, const char *fn,
 				   const char *file, int line);
 
-/* ASSERTION that is safe to use within the debug system */
+/** ASSERTION that is safe to use within the debug system */
 #define __LASSERT(cond)                                                 \
     do {                                                                \
         if (unlikely(!(cond))) {                                        \
@@ -339,3 +393,7 @@ extern void trace_assertion_failed(const
 #endif	/* LUSTRE_TRACEFILE_PRIVATE */
 
 #endif /* __LIBCFS_TRACEFILE_H__ */
+
+/** @} libcfs_tracefile */
+/** @} libcfs_debug */
+/** @} libcfs */
Index: libcfs/libcfs/user-bitops.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/user-bitops.c,v
retrieving revision 1.4
diff -u -p -r1.4 user-bitops.c
--- libcfs/libcfs/user-bitops.c	7 Aug 2008 14:48:51 -0000	1.4
+++ libcfs/libcfs/user-bitops.c	1 Jan 2010 20:03:35 -0000
@@ -40,6 +40,17 @@
 
 #define OFF_BY_START(start)     ((start)/BITS_PER_LONG)
 
+/**
+ * Find first non-zero bit in bitmask pointed by \a addr starting from
+ * a bit specified by \a offset.
+ *
+ * \param addr Points to bitmask
+ * \param size Size of bitmask (in bits)
+ * \param offset # bits to skip prior search
+ *
+ * \return \a size if failed to find non-zero bit, position of
+ * non-zero bit (counting from 0) otherwise
+ */
 unsigned long find_next_bit(unsigned long *addr,
                             unsigned long size, unsigned long offset)
 {
@@ -74,6 +85,17 @@ found:
         return base + bit;
 }
 
+/**
+ * Find first zero bit in bitmask pointed by \a addr starting from
+ * a bit specified by \a offset.
+ *
+ * \param addr Points to bitmask
+ * \param size Size of bitmask (in bits)
+ * \param offset # bits to skip prior search
+ *
+ * \return \a size if failed to find zero bit, position of
+ * zero bit (counting from 0) otherwise
+ */
 unsigned long find_next_zero_bit(unsigned long *addr,
                                  unsigned long size, unsigned long offset)
 {
Index: libcfs/libcfs/user-lock.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/user-lock.c,v
retrieving revision 1.14
diff -u -p -r1.14 user-lock.c
--- libcfs/libcfs/user-lock.c	27 May 2009 02:40:38 -0000	1.14
+++ libcfs/libcfs/user-lock.c	1 Jan 2010 20:03:35 -0000
@@ -42,10 +42,9 @@
 
 /*
  * liblustre is single-threaded, so most "synchronization" APIs are trivial.
- *
- * XXX Liang: There are several branches share lnet with b_hd_newconfig,
- * if we define lock APIs at here, there will be conflict with liblustre
- * in other branches.
+ * However, to test liblustre on commodity h/w, we still need liblustre running
+ * in MT environment. In this case lnet and LNDs need some part of API to be
+ * MT-safe. See the code under HAVE_LIBPTHREAD below.
  */
 
 #ifndef __KERNEL__
Index: libcfs/libcfs/user-mem.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/user-mem.c,v
retrieving revision 1.5
diff -u -p -r1.5 user-mem.c
--- libcfs/libcfs/user-mem.c	7 Oct 2008 04:14:11 -0000	1.5
+++ libcfs/libcfs/user-mem.c	1 Jan 2010 20:03:35 -0000
@@ -53,6 +53,12 @@
  * Allocator
  */
 
+/* XXX Maxim: This is a bug that user-space cfs_alloc_pages returns
+ * only one page for order != 0, not array of pages. Fortunately, nobody
+ * calls it with order !=0. This will be fixed soon by removing
+ * cfs_alloc_pages/cfs_free_pages entirely from libcfs */
+
+/** return page with pre-allocated CFS_PAGE_SIZE * 2^order bytes of memory */
 cfs_page_t *cfs_alloc_pages(int mask, unsigned long order)
 {
         cfs_page_t *pg = malloc(sizeof(*pg));
@@ -127,6 +133,15 @@ void cfs_kunmap(cfs_page_t *pg)
  * SLAB allocator
  */
 
+/**
+ * Create memory cache (SLAB allocator)
+ *
+ * \param name some descriptive name of cache
+ * \param objsize size of objects in this cache
+ * \param off not used
+ * \param flags not used
+ * \return ptr to the cache on success, NULL on failure
+ */
 cfs_mem_cache_t *
 cfs_mem_cache_create(const char *name, size_t objsize, size_t off, unsigned long flags)
 {
@@ -160,9 +175,8 @@ void cfs_mem_cache_free(cfs_mem_cache_t 
 
 /**
  * Returns true if \a addr is an address of an allocated object in a slab \a
- * kmem. Used in assertions. This check is optimistically imprecise, i.e., it
- * occasionally returns true for the incorrect addresses, but if it returns
- * false, then the addresses is guaranteed to be incorrect.
+ * kmem. Used in assertions. Dummy implementation (always return 1) is
+ * OK because it's also imprecise on other platforms.
  */
 int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem)
 {
Index: libcfs/libcfs/user-tcpip.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/user-tcpip.c,v
retrieving revision 1.7
diff -u -p -r1.7 user-tcpip.c
--- libcfs/libcfs/user-tcpip.c	3 Feb 2009 13:43:18 -0000	1.7
+++ libcfs/libcfs/user-tcpip.c	1 Jan 2010 20:03:35 -0000
@@ -82,6 +82,17 @@ libcfs_sock_ioctl(int cmd, unsigned long
         return rc;
 }
 
+/**
+ * Query interface for its IP-address and status (up/down).
+ *
+ * \param name String with name of interface to query
+ * \param up On successful return, status of interface is saved here.
+ * 1 if "up", 0 if "down".
+ * \param ip On successful return, IP-address of interface is saved
+ * here (in host byte-order).
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_ipif_query (char *name, int *up, __u32 *ip)
 {
@@ -143,6 +154,20 @@ libcfs_ipif_free_enumeration (char **nam
         LIBCFS_FREE(names, n * sizeof(*names));
 }
 
+/**
+ * Builds array of names of all known IP-interfaces.
+ *
+ * The caller usually defines a variable of (char **) type and passes
+ * its address as \a namesp. Upon successful return, the variable will
+ * be updated pointing at the array of strings (every slot in the
+ * array is the name of some IP-interface). The array must be freed after
+ * use by means of libcfs_ipif_free_enumeration.
+ *
+ * \param namesp On successful return, the address of array of names
+ * is saved here.
+ *
+ * \return # elements in the array if OK, negative error code otherwise
+ */
 int
 libcfs_ipif_enumerate (char ***namesp)
 {
@@ -237,6 +262,19 @@ libcfs_ipif_enumerate (char ***namesp)
  * Network functions used by user-land lnet acceptor
  */
 
+/**
+ * Create cfs_socket, bind it and start listening.
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the address of newly created
+ * socket is saved here.
+ * \param local_ip IP-address to bind socket to
+ * \param local_port TCP port to bind socket to
+ * \param backlog maxim number of outstanding connection requests allowed
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_listen (cfs_socket_t **sockp,
                     __u32 local_ip, int local_port, int backlog)
@@ -269,6 +307,17 @@ libcfs_sock_release (cfs_socket_t *sock)
         LIBCFS_FREE(sock, sizeof(cfs_socket_t));
 }
 
+/**
+ * Create cfs_socket, accept incoming TCP connection
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param newsockp On successful return, the address of newly created
+ * socket (end-point of newly accepted TCP-connection) is saved here.
+ * \param sock Socket where consumer is listening
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_accept (cfs_socket_t **newsockp, cfs_socket_t *sock)
 {
@@ -294,6 +343,21 @@ libcfs_sock_accept (cfs_socket_t **newso
         return 0;
 }
 
+/**
+ * Read data from the socket
+ *
+ * \param sock Socket to read data from
+ * \param buffer Buffer to store data
+ * \param nob # bytes to read
+ * \param timeout Maximum time interval (in seconds) before return -ETIMEDOUT
+ *
+ * \return 0 if OK, negative error code otherwise
+ *
+ * NOTE: If remote end-point closes connection before \a nob bytes
+ * read, the function returns -EIO.
+ *
+ * \pre \a sock should be in non-blocking mode (libcfs_fcntl_nonblock)
+ */
 int
 libcfs_sock_read (cfs_socket_t *sock, void *buffer, int nob, int timeout)
 {
@@ -341,6 +405,23 @@ libcfs_sock_read (cfs_socket_t *sock, vo
                 return -ETIMEDOUT;
 }
 
+/**
+ * Write data to the socket
+ *
+ * \param sock Socket to write data to
+ * \param buffer Source buffer
+ * \param nob # bytes to write
+ * \param timeout Maximum time interval (in seconds) before return -ETIMEDOUT
+ *
+ * \return 0 if OK, negative error code otherwise
+ *
+ * NOTE: If poll(2) indicated that write is possible (the socket is
+ * not flow-controlled anymore) but write(2) returned zero, the
+ * function regards it as unexpected erroneous situation and returns
+ * -EIO.
+ *
+ * \pre \a sock should be in non-blocking mode (libcfs_fcntl_nonblock)
+ */
 int
 libcfs_sock_write (cfs_socket_t *sock, void *buffer, int nob, int timeout)
 {
@@ -388,7 +469,7 @@ libcfs_sock_write (cfs_socket_t *sock, v
                 return -ETIMEDOUT;
 }
 
-/* Just try to connect to localhost to wake up entity that are
+/** Just try to connect to localhost to wake up entity that are
  * sleeping in accept() */
 void
 libcfs_sock_abort_accept(cfs_socket_t *sock)
@@ -453,6 +534,21 @@ libcfs_sock_getaddr(cfs_socket_t *sock, 
  * Network functions of common use
  */
 
+/**
+ * Create an unnamed pair of connected sockets in the AF_UNIX domain
+ *
+ * Created sockets (\a sockp[0] and \a sockp[1]) must be freed after
+ * use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the addresses of newly created
+ * connected sockets are saved here (in sockp[0] and \a sockp[1]).
+ *
+ * NOTE: It's responsibility of caller to allocate (and free after use
+ * if needed) an array of two pointers to cfs_socket_t. The caller passes
+ * the address of this array as \a sockp argument in this function.
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_socketpair(cfs_socket_t **sockp)
 {
@@ -495,6 +591,7 @@ libcfs_socketpair(cfs_socket_t **sockp)
         return 0;
 }
 
+/** Switch socket to non-blocking mode */
 int
 libcfs_fcntl_nonblock(cfs_socket_t *sock)
 {
@@ -517,6 +614,8 @@ libcfs_fcntl_nonblock(cfs_socket_t *sock
         return 0;
 }
 
+/** Enable Nagle's algorithm on socket. See TCP/IP docs about
+ * TCP_NODELAY option for details */
 int
 libcfs_sock_set_nagle(cfs_socket_t *sock, int nagle)
 {
@@ -540,6 +639,7 @@ libcfs_sock_set_nagle(cfs_socket_t *sock
         return 0;
 }
 
+/** Set both receive and send buffer sizes on socket */
 int
 libcfs_sock_set_bufsiz(cfs_socket_t *sock, int bufsiz)
 {
@@ -568,6 +668,7 @@ libcfs_sock_set_bufsiz(cfs_socket_t *soc
         return 0;
 }
 
+/** Bind given socket to \a ip / \a port */
 int
 libcfs_sock_bind(cfs_socket_t *sock, __u32 ip, __u16 port)
 {
@@ -593,6 +694,24 @@ libcfs_sock_bind(cfs_socket_t *sock, __u
         return 0;
 }
 
+/**
+ * Create TCP socket and bind it to \a local_ip / \a local_port
+ *
+ * The failure to create and bind may be fatal or not as the function
+ * indicates storing one or zero in \a fatal. For now, any failure
+ * related to creation is treated as fatal, while any failure to bind
+ * is treated as not fatal.
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the address of newly created
+ * connected socket is saved here.
+ * \param fatal Points to a flag where "fatality" of operation to be saved
+ * \param local_ip IP-address to bind socket to
+ * \param local_port TCP-port to bind socket to
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_create(cfs_socket_t **sockp, int *fatal,
                    __u32 local_ip, int local_port)
@@ -636,6 +755,19 @@ libcfs_sock_create(cfs_socket_t **sockp,
         return rc;
 }
 
+/**
+ * Connect socket to \a ip / \a port
+ *
+ * The function assumes that \a sock is in non-blocking mode. So, EINPROGRESS
+ * error from connect(2) is treated as "success" and the function returns 0
+ * in this case.
+ *
+ * \param sock Socket to connect
+ * \param ip IP-address of remote peer to connect to
+ * \param port TCP-port of remote peer to connect to
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_connect(cfs_socket_t *sock, __u32 ip, __u16 port)
 {
@@ -661,10 +793,16 @@ libcfs_sock_connect(cfs_socket_t *sock, 
         return 0;
 }
 
-/* NB: EPIPE and ECONNRESET are considered as non-fatal
+/**
+ * Scatter-gather write to socket
+ *
+ * NB: EPIPE and ECONNRESET are considered as non-fatal
  * because:
- * 1) it still makes sense to continue reading &&
- * 2) anyway, poll() will set up POLLHUP|POLLERR flags */
+ * - it still makes sense to continue reading
+ * - anyway, poll() will set up POLLHUP|POLLERR flags
+ *
+ * NB: the function wraps writev in syscall(2) to bypass libsysio
+ */
 int
 libcfs_sock_writev(cfs_socket_t *sock, const struct iovec *vector, int count)
 {
@@ -687,6 +825,11 @@ libcfs_sock_writev(cfs_socket_t *sock, c
         return rc;
 }
 
+/**
+ * Scatter-gather read from socket
+ *
+ * NB: the function wraps readv in syscall(2) to bypass libsysio
+ */
 int
 libcfs_sock_readv(cfs_socket_t *sock, const struct iovec *vector, int count)
 {
Index: libcfs/libcfs/watchdog.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/watchdog.c,v
retrieving revision 1.11
diff -u -p -r1.11 watchdog.c
--- libcfs/libcfs/watchdog.c	21 Jul 2009 17:15:20 -0000	1.11
+++ libcfs/libcfs/watchdog.c	1 Jan 2010 20:03:35 -0000
@@ -43,17 +43,26 @@
 #include <libcfs/libcfs.h>
 #include "tracefile.h"
 
+/** Libcfs watchdog */
 struct lc_watchdog {
-        cfs_timer_t       lcw_timer; /* kernel timer */
+        /** kernel timer */
+        cfs_timer_t       lcw_timer;
+        /** list node to be member of global "pending timers" list */
         struct list_head  lcw_list;
+        /** when (in system ticks) watchdog was touched last time */
         cfs_time_t        lcw_last_touched;
+        /** pointer to the task structure of creator */
         cfs_task_t       *lcw_task;
 
+        /** callback to call when watchdog is triggered */
         void            (*lcw_callback)(pid_t, void *);
+        /** private (consumer-provided) data for \a lcw_callback */
         void             *lcw_data;
 
+        /** process ID of creator */
         pid_t             lcw_pid;
 
+        /** internal state of watchdog */
         enum {
                 LC_WATCHDOG_DISABLED,
                 LC_WATCHDOG_ENABLED,
@@ -100,6 +109,7 @@ static cfs_time_t lcw_last_watchdog_time
 static int lcw_recent_watchdog_count;
 static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED;
 
+/** print out stack-trace of watchdog' task if possible */
 static void
 lcw_dump(struct lc_watchdog *lcw)
 {
@@ -128,6 +138,11 @@ lcw_dump(struct lc_watchdog *lcw)
         EXIT;
 }
 
+/**
+ * Libcfs internal callback. Called by linux kernel when timer
+ * associated with given watchdog got expired.
+ * \param data pointer to libcfs watchdog structure which timer got expired
+ */
 static void lcw_cb(ulong_ptr_t data)
 {
         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
@@ -193,6 +208,12 @@ static void lcw_cb(ulong_ptr_t data)
         EXIT;
 }
 
+/**
+ * Check if any fired watchdog presents or shutdown is in progress.
+ * Used only as condition for conditional wait on lcw_event_waitq.
+ * \return 1 if any fired watchdog presents or shutdown is in
+ * progress, 0 otherwise
+ */
 static int is_watchdog_fired(void)
 {
         int rc;
@@ -206,6 +227,7 @@ static int is_watchdog_fired(void)
         return rc;
 }
 
+/** Stand-alone thread checking for fired watchdogs */
 static int lcw_dispatch_main(void *data)
 {
         int                 rc = 0;
@@ -263,6 +285,10 @@ static int lcw_dispatch_main(void *data)
         RETURN(rc);
 }
 
+/**
+ * Start stand-alone dispatch thread.
+ * Waits until the thread is really started.
+ */
 static void lcw_dispatch_start(void)
 {
         int rc;
@@ -287,6 +313,10 @@ static void lcw_dispatch_start(void)
         EXIT;
 }
 
+/**
+ * Shutdown stand-alone dispatch thread.
+ * Waits until the thread is really shutdown.
+ */
 static void lcw_dispatch_stop(void)
 {
         ENTRY;
@@ -304,6 +334,20 @@ static void lcw_dispatch_stop(void)
         EXIT;
 }
 
+/**
+ * Create and register new watchdog in Libcfs watchdog system.
+ * This is the start point of life-cycle of any watchdog.
+ *
+ * \param timeout not used for now
+ * \param callback user-defined callback to be called when watchdog
+ * got expired or NULL to use default libcfs watchdog callback
+ * \param data private data for user-defined callback
+ *
+ * \return pointer to newly created watchdog
+ *
+ * NOTE: watchdog is created in 'DISABLED' state, should be enabled
+ * explicitly with lc_watchdog_touch.
+ */
 struct lc_watchdog *lc_watchdog_add(int timeout,
                                     void (*callback)(pid_t, void *),
                                     void *data)
@@ -359,6 +403,15 @@ static void lcw_update_time(struct lc_wa
         lcw->lcw_last_touched = newtime;
 }
 
+/**
+ * Enable watchdog and reset its timer. Should be called in two cases:
+ * - to enable watchdog after creation
+ * - to advance expiration point in time
+ *
+ * \param lcw pointer to libcfs watchdog
+ * \param timeout how long (in seconds) expiration time should be
+ * advanced (from 'now')
+ */
 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 {
         ENTRY;
@@ -378,6 +431,11 @@ void lc_watchdog_touch(struct lc_watchdo
 }
 EXPORT_SYMBOL(lc_watchdog_touch);
 
+/**
+ * Disable watchdog pointed by \a lcw. Libcfs won't pay attention to
+ * given watchdog anymore (at least, till it's re-enabled again with
+ * lc_watchdog_touch)
+ */
 void lc_watchdog_disable(struct lc_watchdog *lcw)
 {
         ENTRY;
@@ -395,6 +453,7 @@ void lc_watchdog_disable(struct lc_watch
 }
 EXPORT_SYMBOL(lc_watchdog_disable);
 
+/** De-register \a lcw from libcfs watchdog system and destroy \a lcw */
 void lc_watchdog_delete(struct lc_watchdog *lcw)
 {
         ENTRY;
Index: libcfs/libcfs/linux/linux-curproc.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-curproc.c,v
retrieving revision 1.7
diff -u -p -r1.7 linux-curproc.c
--- libcfs/libcfs/linux/linux-curproc.c	20 Nov 2008 12:51:23 -0000	1.7
+++ libcfs/libcfs/linux/linux-curproc.c	1 Jan 2010 20:03:35 -0000
@@ -76,6 +76,7 @@ pid_t  cfs_curproc_pid(void)
         return current->pid;
 }
 
+/** returns # groups current then current process is member of */
 int    cfs_curproc_groups_nr(void)
 {
         int nr;
@@ -90,6 +91,11 @@ int    cfs_curproc_groups_nr(void)
         return nr;
 }
 
+/**
+ * Store in \a array the gid-s of all groups that current process is member of
+ * \param array where to store gid-s
+ * \param size # gid-s \a array can afford
+ */
 void   cfs_curproc_groups_dump(gid_t *array, int size)
 {
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,4)
@@ -139,6 +145,7 @@ int cfs_cap_raised(cfs_cap_t cap)
         return cap_raised(cfs_current()->cap_effective, cfs_cap_unpack(cap));
 }
 
+/** get capabilities of current process */
 cfs_cap_t cfs_curproc_cap_pack(void)
 {
 #if defined (_LINUX_CAPABILITY_VERSION) && _LINUX_CAPABILITY_VERSION == 0x19980330
@@ -153,6 +160,7 @@ cfs_cap_t cfs_curproc_cap_pack(void)
 #endif
 }
 
+/** set capabilities for current process */
 void cfs_curproc_cap_unpack(cfs_cap_t cap)
 {
 #if defined (_LINUX_CAPABILITY_VERSION) && _LINUX_CAPABILITY_VERSION == 0x19980330
@@ -166,6 +174,7 @@ void cfs_curproc_cap_unpack(cfs_cap_t ca
 #endif
 }
 
+/** check if user is capable to perform actions reflected in \a cap */
 int cfs_capable(cfs_cap_t cap)
 {
         return capable(cfs_cap_unpack(cap));
Index: libcfs/libcfs/linux/linux-fs.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-fs.c,v
retrieving revision 1.3
diff -u -p -r1.3 linux-fs.c
--- libcfs/libcfs/linux/linux-fs.c	7 Aug 2008 14:48:53 -0000	1.3
+++ libcfs/libcfs/linux/linux-fs.c	1 Jan 2010 20:03:35 -0000
@@ -43,6 +43,16 @@
 
 #include <libcfs/libcfs.h>
 
+/**
+ * Open a file by name from kernel
+ *
+ * \param name Path to the file
+ * \param flags Access modes (like O_CREAT, O_EXCL, O_WRONLY, etc.)
+ * \param mode Permissions to use in case of creation (e.g. 0600)
+ * \param err if not NULL, status of operation will be saved there
+ *
+ * \return Libcfs file descriptor on success, NULL otherwise
+ */
 cfs_file_t *
 cfs_filp_open (const char *name, int flags, int mode, int *err)
 {
@@ -65,8 +75,18 @@ cfs_filp_open (const char *name, int fla
 	return filp;
 }
 
-/* write a userspace buffer to disk.
- * NOTE: this returns 0 on success, not the number of bytes written. */
+/**
+ * Write a user-space buffer to disk.
+ *
+ * \param filp Libcfs file descriptor
+ * \param buf Pointer to buffer to write
+ * \param count # bytes of \a buf to write
+ * \param offset Offset from the beginning of file where write will start
+ *
+ * \return  0 on success, negative error code otherwise
+ *
+ * NOTE: this returns 0 on success, not the number of bytes written.
+ */
 ssize_t
 cfs_user_write (cfs_file_t *filp, const char *buf, size_t count, loff_t *offset)
 {
Index: libcfs/libcfs/linux/linux-kernelcomm.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-kernelcomm.c,v
retrieving revision 1.5
diff -u -p -r1.5 linux-kernelcomm.c
--- libcfs/libcfs/linux/linux-kernelcomm.c	26 Oct 2009 08:08:25 -0000	1.5
+++ libcfs/libcfs/linux/linux-kernelcomm.c	1 Jan 2010 20:03:35 -0000
@@ -64,8 +64,10 @@ static struct sock *lnl_socket = NULL;
 static atomic_t lnl_start_count = ATOMIC_INIT(0);
 static spinlock_t lnl_lock = SPIN_LOCK_UNLOCKED;
 
-/** Start the netlink socket for this transport
- * @param transport lnl_transport
+/**
+ * Start the netlink socket for this transport
+ * \param transport lnl_transport
+ * \return 0 if OK, negative error value otherwise
  */
 int libcfs_klnl_start(int transport)
 {
@@ -115,9 +117,13 @@ static void send_shutdown_msg(int transp
         libcfs_klnl_msg_put(0, group, &lh);
 }
 
-/* This should be called once per (started) transport
- * @param transport lnl_transport
- * @param group Broadcast group for shutdown message */
+/**
+ * Stop the netlink socket for this transport
+ *  This should be called once per (started) transport
+ * \param transport lnl_transport
+ * \param group Broadcast group for shutdown message
+ * \return 0 (always)
+ */
 int libcfs_klnl_stop(int transport, int group)
 {
         if (group)
@@ -164,11 +170,12 @@ static struct sk_buff *netlink_make_msg(
 }
 
 /**
- * libcfs_klnl_msg_put - send an message from kernel to userspace
- * @param pid Process id to send message to for unicast messages; must be 0 for
+ * Send an message from kernel to user-space
+ * \param pid Process id to send message to for unicast messages; must be 0 for
  *   broadcast
- * @param group Broadcast group; 0 for unicast messages
- * @param payload Payload data.  First field of payload is always struct lnl_hdr
+ * \param group Broadcast group; 0 for unicast messages
+ * \param payload Payload data.  First field of payload is always struct lnl_hdr
+ * \return 0 if OK, negative error value otherwise
  *
  * Allocates an skb, builds the netlink message, and sends it to the pid.
  */
Index: libcfs/libcfs/linux/linux-mem.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-mem.c,v
retrieving revision 1.6
diff -u -p -r1.6 linux-mem.c
--- libcfs/libcfs/linux/linux-mem.c	26 Oct 2009 08:08:26 -0000	1.6
+++ libcfs/libcfs/linux/linux-mem.c	1 Jan 2010 20:03:35 -0000
@@ -41,6 +41,12 @@
 #include <linux/highmem.h>
 #include <libcfs/libcfs.h>
 
+/**
+ * Convert libcfs alloc flags (defined in enum cfs_alloc_flags)
+ * to linux native representation.
+ * \param flags Flag combination to convert
+ * \return linux representation of \a flag
+ */
 static unsigned int cfs_alloc_flags_to_gfp(u_int32_t flags)
 {
 	unsigned int mflags = 0;
@@ -100,6 +106,7 @@ cfs_free_large(void *addr)
 	vfree(addr);
 }
 
+/** return array of 2^order pages */
 cfs_page_t *cfs_alloc_pages(unsigned int flags, unsigned int order)
 {
         /*
@@ -114,6 +121,15 @@ void __cfs_free_pages(cfs_page_t *page, 
         __free_pages(page, order);
 }
 
+/**
+ * Create memory cache (SLAB allocator)
+ *
+ * \param name some descriptive name of cache
+ * \param size size of objects in this cache
+ * \param offset required alignment for the objects
+ * \param flags SLAB flags (like SLAB_HWCACHE_ALIGN)
+ * \return ptr to the cache on success, NULL on failure
+ */
 cfs_mem_cache_t *
 cfs_mem_cache_create (const char *name, size_t size, size_t offset,
                       unsigned long flags)
Index: libcfs/libcfs/linux/linux-prim.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-prim.c,v
retrieving revision 1.10
diff -u -p -r1.10 linux-prim.c
--- libcfs/libcfs/linux/linux-prim.c	26 Oct 2009 08:08:26 -0000	1.10
+++ libcfs/libcfs/linux/linux-prim.c	1 Jan 2010 20:03:35 -0000
@@ -225,10 +225,13 @@ void cfs_daemonize(char *str) {
 
         lock_kernel();
         daemonize(str);
+
+        /* block all signals */
         SIGNAL_MASK_LOCK(current, flags);
         sigfillset(&current->blocked);
         RECALC_SIGPENDING;
         SIGNAL_MASK_UNLOCK(current, flags);
+
         unlock_kernel();
 }
 
@@ -236,7 +239,13 @@ int cfs_daemonize_ctxt(char *str) {
         struct task_struct *tsk = current;
         struct fs_struct *fs = NULL;
 
+        /* among other things, re-parents tsk->fs to init's fs structure */
         cfs_daemonize(str);
+
+        /*
+         * clone init's fs struct to have a local copy that caller
+         * can safely modify afterwards
+         */
         fs = copy_fs_struct(tsk->fs);
         if (fs == NULL)
                 return -ENOMEM;
Index: libcfs/libcfs/linux/linux-tcpip.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/linux/linux-tcpip.c,v
retrieving revision 1.5
diff -u -p -r1.5 linux-tcpip.c
--- libcfs/libcfs/linux/linux-tcpip.c	26 Oct 2009 08:08:26 -0000	1.5
+++ libcfs/libcfs/linux/linux-tcpip.c	1 Jan 2010 20:03:35 -0000
@@ -102,6 +102,19 @@ libcfs_sock_ioctl(int cmd, unsigned long
         return rc;
 }
 
+/**
+ * Query interface for its IP-address and status (up/down).
+ *
+ * \param name String with name of interface to query
+ * \param up On successful return, status of interface is saved here.
+ * 1 if "up", 0 if "down".
+ * \param ip On successful return, IP-address of interface is saved
+ * here (in host byte-order).
+ * \param mask On successful return, netmask of interface is saved
+ * here (in host byte-order).
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_ipif_query (char *name, int *up, __u32 *ip, __u32 *mask)
 {
@@ -164,6 +177,20 @@ libcfs_ipif_query (char *name, int *up, 
 
 EXPORT_SYMBOL(libcfs_ipif_query);
 
+/**
+ * Builds array of names of all known IP-interfaces.
+ *
+ * The caller usually defines a variable of (char **) type and passes
+ * its address as \a namesp. Upon successful return, the variable will
+ * be updated pointing at the array of strings (every slot in the
+ * array is the name of some IP-interface). The array must be freed after
+ * use by means of libcfs_ipif_free_enumeration.
+ *
+ * \param namesp On successful return, the address of array of names
+ * is saved here.
+ *
+ * \return # elements in the array if OK, negative error code otherwise
+ */
 int
 libcfs_ipif_enumerate (char ***namesp)
 {
@@ -279,6 +306,20 @@ libcfs_ipif_free_enumeration (char **nam
 
 EXPORT_SYMBOL(libcfs_ipif_free_enumeration);
 
+/**
+ * Write data to the socket
+ *
+ * \param sock Socket to write data to
+ * \param buffer Source buffer
+ * \param nob # bytes to write
+ * \param timeout Maximum time interval (in seconds) before return -ETIMEDOUT
+ *
+ * \return 0 if OK, negative error code otherwise
+ *
+ * NOTE: The function requests blocking send with timeout
+ * specified. If send returns zero in spite of it, the function treats
+ * such situation as erroneous and returns -ECONNABORTED.
+ */
 int
 libcfs_sock_write (struct socket *sock, void *buffer, int nob, int timeout)
 {
@@ -353,6 +394,19 @@ libcfs_sock_write (struct socket *sock, 
 }
 EXPORT_SYMBOL(libcfs_sock_write);
 
+/**
+ * Read data from the socket
+ *
+ * \param sock Socket to read data from
+ * \param buffer Buffer to store data
+ * \param nob # bytes to read
+ * \param timeout Maximum time interval (in seconds) before return -ETIMEDOUT
+ *
+ * \return 0 if OK, negative error code otherwise
+ *
+ * NOTE: If remote end-point closes connection before \a nob bytes
+ * read, the function returns -ECONNRESET.
+ */
 int
 libcfs_sock_read (struct socket *sock, void *buffer, int nob, int timeout)
 {
@@ -420,6 +474,24 @@ libcfs_sock_read (struct socket *sock, v
 
 EXPORT_SYMBOL(libcfs_sock_read);
 
+/**
+ * Create TCP socket and bind it to \a local_ip / \a local_port
+ *
+ * The failure to create and bind may be fatal or not as the function
+ * indicates storing one or zero in \a fatal. For now, any failure
+ * related to creation is treated as fatal, while "EADDRINUSE" failure
+ * to bind is treated as not fatal.
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the address of newly created
+ * connected socket is saved here.
+ * \param fatal Points to a flag where "fatality" of operation to be saved
+ * \param local_ip IP-address to bind socket to
+ * \param local_port TCP-port to bind socket to
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 static int
 libcfs_sock_create (struct socket **sockp, int *fatal,
                     __u32 local_ip, int local_port)
@@ -516,6 +588,11 @@ libcfs_sock_setbuf (struct socket *sock,
 
 EXPORT_SYMBOL(libcfs_sock_setbuf);
 
+/**
+ * If \a remote is zero, stores ip/port of local end-point of
+ * TCP-connection in \a ip / \a port. Otherwise, ip/port of
+ * remote end.
+ */
 int
 libcfs_sock_getaddr (struct socket *sock, int remote, __u32 *ip, int *port)
 {
@@ -559,6 +636,19 @@ libcfs_sock_getbuf (struct socket *sock,
 
 EXPORT_SYMBOL(libcfs_sock_getbuf);
 
+/**
+ * Create cfs_socket, bind it and start listening.
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the address of newly created
+ * socket is saved here.
+ * \param local_ip IP-address to bind socket to
+ * \param local_port TCP port to bind socket to
+ * \param backlog maxim number of outstanding connection requests allowed
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_listen (struct socket **sockp,
                     __u32 local_ip, int local_port, int backlog)
@@ -601,6 +691,17 @@ int sock_create_lite(int family, int typ
 }
 #endif
 
+/**
+ * Create socket, accept incoming TCP connection
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param newsockp On successful return, the address of newly created
+ * socket (end-point of newly accepted TCP-connection) is saved here.
+ * \param sock Socket where consumer is listening
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_accept (struct socket **newsockp, struct socket *sock)
 {
@@ -646,6 +747,7 @@ libcfs_sock_accept (struct socket **news
 
 EXPORT_SYMBOL(libcfs_sock_accept);
 
+/** wake up thread sleeping in accept(\a sock) */
 void
 libcfs_sock_abort_accept (struct socket *sock)
 {
@@ -654,6 +756,27 @@ libcfs_sock_abort_accept (struct socket 
 
 EXPORT_SYMBOL(libcfs_sock_abort_accept);
 
+/**
+ * Create TCP socket, bind it to \a local_ip / \a local_port and
+ * connect it to \a peer_ip / \a peer_port
+ *
+ * The failure to create/bind/connect may be fatal or not as the function
+ * indicates storing one or zero in \a fatal. The most types of
+ * failures are treated as fatal. The only exception is when bind or
+ * connect fails because of "port already in use" situation.
+ *
+ * Created socket must be freed after use by means of libcfs_sock_release.
+ *
+ * \param sockp On successful return, the address of newly created, bound
+ * and connected socket is saved here.
+ * \param fatal Points to a flag where "fatality" of operation to be saved
+ * \param local_ip IP-address to bind socket to
+ * \param local_port TCP-port to bind socket to
+ * \param peer_ip IP-address of remote peer to connect socket to
+ * \param peer_port TCP-port of remote peer to connect socket to
+ *
+ * \return 0 if OK, negative error code otherwise
+ */
 int
 libcfs_sock_connect (struct socket **sockp, int *fatal,
                      __u32 local_ip, int local_port,
Index: libcfs/libcfs/ulinux/ulinux-kernelcomm.c
===================================================================
RCS file: /cvsroot/cfs/libcfs/libcfs/ulinux/ulinux-kernelcomm.c,v
retrieving revision 1.4
diff -u -p -r1.4 ulinux-kernelcomm.c
--- libcfs/libcfs/ulinux/ulinux-kernelcomm.c	19 Sep 2009 18:36:18 -0000	1.4
+++ libcfs/libcfs/ulinux/ulinux-kernelcomm.c	1 Jan 2010 20:03:35 -0000
@@ -53,10 +53,12 @@
 #include <sys/socket.h>
 #include <linux/netlink.h>
 
-/** Start the userspace side of a LNL pipe.
- * @param link Private descriptor for pipe/socket.
- * @param groups LNL broadcast group to listen to
+/**
+ * Start the user-space side of a LNL pipe.
+ * \param link Private descriptor for pipe/socket.
+ * \param groups LNL broadcast group to listen to
  *          (can be null for unicast to this pid)
+ * \return 0 if OK, negative error value otherwise
  */
 int libcfs_ulnl_start(lustre_netlink *link, int groups)
 {
@@ -86,14 +88,16 @@ int libcfs_ulnl_stop(lustre_netlink *lin
         return close(*link);
 }
 
-/** Read a message from the netlink layer.
+/**
+ * Read a message from the netlink layer.
  * Allocates memory, returns handle
  *
- * @param link Private descriptor for pipe/socket.
- * @param maxsize Maximum message size allowed
- * @param transport Only listen to messages on this transport
+ * \param link Private descriptor for pipe/socket.
+ * \param maxsize Maximum message size allowed
+ * \param transport Only listen to messages on this transport
  *      (and the generic transport)
- * @param lnlhh Handle to the new LNL message
+ * \param lnlhh Handle to the new LNL message
+ * \return 0 if OK, negative error value otherwise
  */
 int libcfs_ulnl_msg_get(lustre_netlink *link, int maxsize, int transport,
                         struct lnl_hdr **lnlhh)
@@ -149,7 +153,7 @@ int libcfs_ulnl_msg_get(lustre_netlink *
         return rc;
 }
 
-/* Free a message returned by the above fn */
+/** Free a message returned by the above fn */
 int libcfs_ulnl_msg_free(struct lnl_hdr **lnlhh)
 {
         /* compute nlmsdghdr offset */
