/*
 * Copyright (c) 2001-2005, Raphael Manfredi
 * Copyright (c) 2000 Daniel Walker (dwalker@cats.ucsc.edu)
 *
 *----------------------------------------------------------------------
 * This file is part of gtk-gnutella.
 *
 *  gtk-gnutella is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  gtk-gnutella is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with gtk-gnutella; if not, write to the Free Software
 *  Foundation, Inc.:
 *      59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *----------------------------------------------------------------------
 */

/**
 * @ingroup core
 * @file
 *
 * Handle sharing of our own files and answers to remote queries.
 *
 * @author Daniel Walker (dwalker@cats.ucsc.edu)
 * @date 2000
 * @author Raphael Manfredi
 * @date 2001-2005
 */

#include "common.h"

#include "share.h"
#include "extensions.h"
#include "downloads.h"
#include "fileinfo.h"
#include "ggep_type.h"
#include "gmsg.h"
#include "gnet_stats.h"
#include "guid.h"
#include "hosts.h"
#include "huge.h"
#include "nodes.h"
#include "oob.h"
#include "oob_proxy.h"
#include "publisher.h"
#include "qhit.h"
#include "qrp.h"
#include "search.h"
#include "settings.h"
#include "spam.h"
#include "upload_stats.h"
#include "uploads.h"

#include "if/gnet_property.h"
#include "if/gnet_property_priv.h"
#include "if/bridge/c2ui.h"

#include "lib/ascii.h"
#include "lib/atoms.h"
#include "lib/bg.h"
#include "lib/cq.h"
#include "lib/endian.h"
#include "lib/file.h"
#include "lib/glib-missing.h"
#include "lib/halloc.h"
#include "lib/hashlist.h"
#include "lib/listener.h"
#include "lib/mime_type.h"
#include "lib/str.h"
#include "lib/tm.h"
#include "lib/utf8.h"
#include "lib/walloc.h"


#include "lib/override.h"		/* Must be the last header included */

#define SHARE_RECENT_THRESH		(2 * 7 * 24 * 60 * 60)	/* 2 weeks */

enum shared_file_magic {
	SHARED_FILE_MAGIC = 0x3702b437U
};

struct shared_file {
	enum shared_file_magic magic;

	struct dl_file_info *fi;	/**< PFSP-server: the holding fileinfo */
	const struct sha1 *sha1;	/**< SHA1 digest, binary form, atom */
	const struct tth *tth;		/**< TTH digest, binary form, atom */

	const char *file_path;		/**< The full path of the file (atom!) */
	const char *name_nfc;		/**< UTF-8 NFC version of filename (atom!) */
	const char *name_canonic;	/**< UTF-8 canonized ver. of filename (atom)! */
	const char *relative_path;	/**< UTF-8 NFC string (atom) */

	size_t name_nfc_len;		/**< strlen(name_nfc) */
	size_t name_canonic_len;	/**< strlen(name_canonic) */

	time_t mtime;				/**< Last modif. time, for SHA1 computation */

	filesize_t file_size;		/**< File size in Bytes */
	guint32 file_index;			/**< the files index within our local DB */
	guint32 sort_index;			/**< the index for sorted listings */

	enum mime_type mime_type;	/* MIME type of the file */

	int refcnt;					/**< Reference count */
	guint32 flags;				/**< See below for definition */
};

/**
 * Describes special files which are served by GTKG.
 */
struct special_file {
	const char *path;			/* URL path */
	const char *file;			/* File name to load from disk */
	enum mime_type type;		/* MIME type of the file */
	const char *what;			/* Description of the file for traces */
};

static const struct special_file specials[] = {
	{ "/favicon.ico",
			"favicon.png",	MIME_TYPE_IMAGE_PNG,	"Favorite web icon" },
	{ "/robots.txt",
			"robots.txt",	MIME_TYPE_TEXT_PLAIN,	"Robot exclusion" },
};

/**
 * Maps special names (e.g. "/favicon.ico") to the shared_file_t structure.
 */
static GHashTable *special_names;

static GHashTable *extensions;	/* Shared filename extensions */
static GSList *shared_dirs;
static hash_list_t *partial_files;
static cevent_t *share_qrp_rebuild_ev;

/*
 * These variables are recreated by each library scanning.
 *
 * During a rebuild of the library, their old values remain present so that
 * we can continue to serve files without interruption.  Only at the end of
 * the rebuilding process do we atomically update all of them with the new
 * values, freeing old content.
 */
static guint64 files_scanned;	/* Amount of files shared in the library */
static guint64 bytes_scanned;
static GSList *shared_files;
static search_table_t *search_table;
static GHashTable *file_basenames;
static search_table_t *partial_table;
static shared_file_t **file_table;			/* Sorted by mtime */
static shared_file_t **sorted_file_table;	/* Sorted by name */

static struct recursive_scan *recursive_scan_context;
static gboolean share_rebuilding;

/**
 * This tree maps a SHA1 hash (base-32 encoded) onto the corresponding
 * shared_file if we have one.
 */
static GTree *sha1_to_share;

#define A	SEARCH_AUDIO_TYPE
#define V	SEARCH_VIDEO_TYPE
#define D	SEARCH_DOC_TYPE
#define I	SEARCH_IMG_TYPE
#define U	(SEARCH_WIN_TYPE | SEARCH_UNIX_TYPE)

/**
 * This table encodes for each known MIME type the searchable media type bits
 * that are applicable for that type.
 *
 * A 0 (zero) entry means the corresponding MIME type will never match any
 * of the searchable bits.  This typically indicates a file containing
 * either application-specific data (e.g. an MP3 playlist), a programming
 * language (e.g. a C header file), or binary-specific data (e.g. a ROM
 * image).
 */
static struct {
	enum mime_type type;
	unsigned flags;
} media_type_map[] = {
	/* NOTE: this is mapped to a hash, but keep sorted for easier updates */
	{ MIME_TYPE_APPLICATION_7Z,						U },
	{ MIME_TYPE_APPLICATION_ACE,					U },
	{ MIME_TYPE_APPLICATION_ANDROID_PACKAGE,		U },
	{ MIME_TYPE_APPLICATION_BITTORRENT,				D },
	{ MIME_TYPE_APPLICATION_BROADBAND_EBOOK,		D },
	{ MIME_TYPE_APPLICATION_BZIP2,					U },
	{ MIME_TYPE_APPLICATION_COMPILED_HTML_HELP,		D },
	{ MIME_TYPE_APPLICATION_COMPRESS,				U },
	{ MIME_TYPE_APPLICATION_DEB,					U },
	{ MIME_TYPE_APPLICATION_DMG,					U },
	{ MIME_TYPE_APPLICATION_DOSEXEC,				U },
	{ MIME_TYPE_APPLICATION_EPUB,					D },
	{ MIME_TYPE_APPLICATION_EXCEL,					0 },
	{ MIME_TYPE_APPLICATION_GAMEBOY_ROM,			0 },
	{ MIME_TYPE_APPLICATION_GENESIS_ROM,			0 },
	{ MIME_TYPE_APPLICATION_GZIP,					U },
	{ MIME_TYPE_APPLICATION_IPHONE_APP,				0 },
	{ MIME_TYPE_APPLICATION_ISO9660,				U },
	{ MIME_TYPE_APPLICATION_JAR,					U },
	{ MIME_TYPE_APPLICATION_JAVASCRIPT,				U|D },
	{ MIME_TYPE_APPLICATION_JAVA_SER,				0 },
	{ MIME_TYPE_APPLICATION_JAVA_VM,				0 },
	{ MIME_TYPE_APPLICATION_LYX,					D },
	{ MIME_TYPE_APPLICATION_LZH,					U },
	{ MIME_TYPE_APPLICATION_MOBIPOCKET_EBOOK,		D },
	{ MIME_TYPE_APPLICATION_MSWORD,					D },
	{ MIME_TYPE_APPLICATION_MS_READER,				D },
	{ MIME_TYPE_APPLICATION_MS_SHORTCUT,			0 },
	{ MIME_TYPE_APPLICATION_N64_ROM,				0 },
	{ MIME_TYPE_APPLICATION_NES_ROM,				0 },
	{ MIME_TYPE_APPLICATION_OBJECT,					0 },
	{ MIME_TYPE_APPLICATION_OGG,					A },
	{ MIME_TYPE_APPLICATION_OPEN_PACKAGING_FORMAT,	U },
	{ MIME_TYPE_APPLICATION_PDF,					D },
	{ MIME_TYPE_APPLICATION_POSTSCRIPT,				D|I },
	{ MIME_TYPE_APPLICATION_POWERPOINT,				D },
	{ MIME_TYPE_APPLICATION_RAR,					U },
	{ MIME_TYPE_APPLICATION_RDF,					D },
	{ MIME_TYPE_APPLICATION_RSS,					D },
	{ MIME_TYPE_APPLICATION_SH,						U },
	{ MIME_TYPE_APPLICATION_SHAR,					U },
	{ MIME_TYPE_APPLICATION_SHOCKWAVE_FLASH,		V },
	{ MIME_TYPE_APPLICATION_SIT,					U },
	{ MIME_TYPE_APPLICATION_SNES_ROM,				0 },
	{ MIME_TYPE_APPLICATION_TAR,					U },
	{ MIME_TYPE_APPLICATION_TEX,					D },
	{ MIME_TYPE_APPLICATION_TEXINFO,				D },
	{ MIME_TYPE_APPLICATION_TROFF,					D },
	{ MIME_TYPE_APPLICATION_TROFF_MAN,				D },
	{ MIME_TYPE_APPLICATION_TROFF_ME,				D },
	{ MIME_TYPE_APPLICATION_TROFF_MS,				D },
	{ MIME_TYPE_APPLICATION_ZIP,					U },
	{ MIME_TYPE_AUDIO_BASIC,						A },
	{ MIME_TYPE_AUDIO_FLAC,							A },
	{ MIME_TYPE_AUDIO_MATROSKA,						A },
	{ MIME_TYPE_AUDIO_MIDI,							A },
	{ MIME_TYPE_AUDIO_MP4,							A },
	{ MIME_TYPE_AUDIO_MPEG,							A },
	{ MIME_TYPE_AUDIO_MPEGURL,						0 },
	{ MIME_TYPE_AUDIO_MS_ASF,						A },
	{ MIME_TYPE_AUDIO_OGG,							A },
	{ MIME_TYPE_AUDIO_PLAYLIST,						0 },
	{ MIME_TYPE_AUDIO_REALAUDIO,					A },
	{ MIME_TYPE_AUDIO_SPEEX,						A },
	{ MIME_TYPE_AUDIO_WAVE,							A },
	{ MIME_TYPE_IMAGE_BMP,							I },
	{ MIME_TYPE_IMAGE_GIF,							I },
	{ MIME_TYPE_IMAGE_JPEG,							I },
	{ MIME_TYPE_IMAGE_PNG,							I },
	{ MIME_TYPE_IMAGE_PSD,							I },
	{ MIME_TYPE_IMAGE_TGA,							I },
	{ MIME_TYPE_IMAGE_TIFF,							I },
	{ MIME_TYPE_IMAGE_XPM,							I },
	{ MIME_TYPE_MESSAGE_RFC822,						D },
	{ MIME_TYPE_TEXT_C,								0 },
	{ MIME_TYPE_TEXT_CALENDAR,						0 },
	{ MIME_TYPE_TEXT_CHDR,							0 },
	{ MIME_TYPE_TEXT_CPP,							0 },
	{ MIME_TYPE_TEXT_CPPHDR,						0 },
	{ MIME_TYPE_TEXT_CSS,							0 },
	{ MIME_TYPE_TEXT_CSV,							D },
	{ MIME_TYPE_TEXT_DIFF,							0 },
	{ MIME_TYPE_TEXT_HTML,							D },
	{ MIME_TYPE_TEXT_JAVA,							0 },
	{ MIME_TYPE_TEXT_LATEX,							D },
	{ MIME_TYPE_TEXT_LILYPOND,						D },
	{ MIME_TYPE_TEXT_PERL,							0 },
	{ MIME_TYPE_TEXT_PLAIN,							D },
	{ MIME_TYPE_TEXT_PYTHON,						0 },
	{ MIME_TYPE_TEXT_RTF,							D },
	{ MIME_TYPE_TEXT_XHTML,							D },
	{ MIME_TYPE_TEXT_XML,							D },
	{ MIME_TYPE_VIDEO_FLV,							V },
	{ MIME_TYPE_VIDEO_MATROSKA,						V },
	{ MIME_TYPE_VIDEO_MP4,							V },
	{ MIME_TYPE_VIDEO_MPEG,							V },
	{ MIME_TYPE_VIDEO_MSVIDEO,						V },
	{ MIME_TYPE_VIDEO_MS_ASF,						V },
	{ MIME_TYPE_VIDEO_OGG,							V },
	{ MIME_TYPE_VIDEO_OGM,							V },
	{ MIME_TYPE_VIDEO_QUICKTIME,					V },
};

#undef A
#undef V
#undef D
#undef I
#undef U

/**
 * Hash table yielding the media type flags from a MIME type.
 * Built dynamically from media_type_map[].
 */
static GHashTable *share_media_types;

/**
 * Compare binary SHA1 hashes.
 * @return 0 if they're the same, a negative or positive number if s1 if greater
 * than s2 or s1 greater than s2, respectively.
 * Used to search the sha1_to_share tree.
 */
static int
compare_share_sha1(gconstpointer s1, gconstpointer s2)
{
	return memcmp(s1, s2, SHA1_RAW_SIZE);
}

/**
 * Reset sha1_to_share
 */
static void
reinit_sha1_table(void)
{
	if (sha1_to_share)
		g_tree_destroy(sha1_to_share);

	sha1_to_share = g_tree_new(compare_share_sha1);
}

void
shared_file_check(const shared_file_t *sf)
{
	g_assert(sf);
	g_assert(SHARE_REBUILDING != sf);
	g_assert(SHARED_FILE_MAGIC == sf->magic);
	g_assert(sf->refcnt >= 0);
	g_assert((NULL != sf->name_nfc) ^ (0 == sf->name_nfc_len));
	g_assert((NULL != sf->name_canonic) ^ (0 == sf->name_canonic_len));
}

/**
 * Allocate a shared_file_t structure.
 */
static shared_file_t *
shared_file_alloc(void)
{
	static const shared_file_t zero_sf;
	shared_file_t *sf;

	WALLOC(sf);
	*sf = zero_sf;
	sf->magic = SHARED_FILE_MAGIC;
	return sf;
}

static void
shared_file_deindex(shared_file_t *sf)
{
	shared_file_check(sf);

	if (SHARE_F_BASENAME & sf->flags) {
		if (file_basenames != NULL) {
			g_hash_table_remove(file_basenames, sf->name_nfc);
		}
	}
	sf->flags &= ~SHARE_F_BASENAME;

	/*
	 * The shared file might not be referenced by the current file_table
	 * either because it hasn't been build yet or because of a rescan.
	 */

	if (
		file_table &&
		sf->file_index > 0 &&
		sf->file_index <= files_scanned &&
		sf == file_table[sf->file_index - 1]
   ) {
		g_assert(SHARE_F_INDEXED & sf->flags);
		file_table[sf->file_index - 1] = NULL;
	}
	if (
		sorted_file_table &&
		sf->sort_index > 0 &&
		sf->sort_index <= files_scanned &&
		sf == sorted_file_table[sf->sort_index - 1]
   ) {
		g_assert(SHARE_F_INDEXED & sf->flags);
		sorted_file_table[sf->sort_index - 1] = NULL;
	}
	sf->file_index = 0;
	sf->sort_index = 0;
	sf->flags &= ~SHARE_F_INDEXED;

	/*
	 * Shared file is no longer indexed so it no longer belongs to the
	 * shared set and needs to be removed if it was referenced there.
	 */

	if (sf->sha1 != NULL) {
		shared_file_t *current;
		gpointer key;

		key = deconstify_gpointer(sf->sha1);
		current = g_tree_lookup(sha1_to_share, key);
		if (current == sf) {
			g_tree_remove(sha1_to_share, key);
		}
	}
}

/**
 * Dispose of a shared_file_t structure and nullify the pointer.
 */
static void
shared_file_free(shared_file_t **sf_ptr)
{
	g_assert(sf_ptr);
	if (*sf_ptr) {
		shared_file_t *sf = *sf_ptr;

		g_assert(sf->refcnt == 0);

		if (sf->flags & SHARE_F_INDEXED)
			shared_file_deindex(sf);

		atom_sha1_free_null(&sf->sha1);
		atom_tth_free_null(&sf->tth);
		atom_str_free_null(&sf->relative_path);
		atom_str_free_null(&sf->file_path);
		atom_str_free_null(&sf->name_nfc);
		atom_str_free_null(&sf->name_canonic);
		sf->magic = 0;

		WFREE(sf);
		*sf_ptr = NULL;
	}
}

/**
 * Set canonic, NFC and NFKC normalized names.
 *
 * @return whether an error occurred
 */
static gboolean
shared_file_set_names(shared_file_t *sf, const char *filename)
{
  	shared_file_check(sf);	
   	g_assert(!sf->name_nfc);
   	g_assert(!sf->name_canonic);

	/* Set the NFC normalized name. */
	{	
		char *name = filename_to_utf8_normalized(filename, UNI_NORM_NETWORK);
		sf->name_nfc = atom_str_get(name);
		G_FREE_NULL(name);
	}

	/*
	 * Set the NFKC normalized name. Also prepend the relative path
	 * if enabled. Queries will be matched against this string.
	 */
	{
		char *name, *name_canonic;

		if (
			GNET_PROPERTY(search_results_expose_relative_paths) &&
			sf->relative_path
		) {
			name = g_strconcat(sf->relative_path, " ", sf->name_nfc,
						(void *) 0);
		} else {
			name = deconstify_gchar(sf->name_nfc);
		}
		name_canonic = UNICODE_CANONIZE(name);
		sf->name_canonic = atom_str_get(name_canonic);
		if (name_canonic != name) {
			HFREE_NULL(name_canonic);
		}
		if (name != sf->name_nfc) {
			G_FREE_NULL(name);
		}
	}

	sf->name_nfc_len = strlen(sf->name_nfc);
	sf->name_canonic_len = strlen(sf->name_canonic);

	if (0 == sf->name_nfc_len || 0 == sf->name_canonic_len) {
		g_warning("Normalized filename is an empty string \"%s\" "
			"(NFC=\"%s\", canonic=\"%s\")",
			filename, sf->name_nfc, sf->name_canonic);
		return TRUE;
	}
	return FALSE;
}

static const guint FILENAME_CLASH = -1;		/**< Indicates basename clashes */
static const guint PARTIAL_FILE = -2;		/**< Indicates partial file */
static const guint SPECIAL_FILE = -3;		/**< Special served files */

/**
 * Initialize special file entry, returning shared_file_t structure if
 * the file exists, NULL otherwise.
 */
static G_GNUC_COLD shared_file_t *
share_special_load(const struct special_file *sp)
{
	FILE *f;
	int idx = 0;
	char *tmp;
	shared_file_t *sf = NULL;
	file_path_t fp[4];
	unsigned length = 0;

	tmp = get_folder_path(PRIVLIB_PATH, NULL);
	if (tmp != NULL)
		file_path_set(&fp[length++], tmp, sp->file);

	file_path_set(&fp[length++], settings_config_dir(), sp->file);
	file_path_set(&fp[length++], PRIVLIB_EXP, sp->file);
#ifndef OFFICIAL_BUILD
	file_path_set(&fp[length++], PACKAGE_EXTRA_SOURCE_DIR, sp->file);
#endif

	g_assert(length <= G_N_ELEMENTS(fp));
	
	f = file_config_open_read_norename_chosen(sp->what, fp, length, &idx);

	if (NULL == f)
		goto done;

	/*
	 * Create fake special file sharing structure, so that we can
	 * upload it if requested.
	 */

	sf = shared_file_alloc();
	sf->flags |= SHARE_F_SPECIAL;
	sf->file_index = SPECIAL_FILE;

	{
		char *filename = make_pathname(fp[idx].dir, fp[idx].name);
		sf->file_path = atom_str_get(filename);
		HFREE_NULL(filename);
	}
	if (shared_file_set_names(sf, sp->file)) {
		shared_file_free(&sf);
	} else {
		sf->mime_type = sp->type;
	}

	fclose(f);

done:
	HFREE_NULL(tmp);

	return sf;
}

void
shared_files_match(const char *query,
	st_search_callback callback, gpointer user_data,
	int max_res, gboolean partials, query_hashvec_t *qhv)
{
	int n;
	int remain;

	/*
	 * First search from the library.
	 */

	n = st_search(search_table, query, callback, user_data, max_res, qhv);
	gnet_stats_count_general(GNR_LOCAL_HITS, n);

	remain = max_res - n;

	/*
	 * Then if we still can supply some hits, look whether we have a partial
	 * file matching.
	 *
	 * Matching on partials is done only when users request that explicitly
	 * in their query (through the GGEP "PR" key) and when we serve partial
	 * files (PFSP server) and they configured answering to partial requests.
	 */

	if (partials && remain > 0 && share_can_answer_partials()
	) {
		n = st_search(partial_table, query, callback, user_data, remain, NULL);
		gnet_stats_count_general(GNR_LOCAL_PARTIAL_HITS, n);
	}
}

/**
 * Initialize the special files we're sharing.
 */
static G_GNUC_COLD void
share_special_init(void)
{
	guint i;

	special_names = g_hash_table_new(g_str_hash, g_str_equal);

	for (i = 0; i < G_N_ELEMENTS(specials); i++) {
		shared_file_t *sf = share_special_load(&specials[i]);
		if (sf != NULL)
			g_hash_table_insert(special_names,
				deconstify_gchar(specials[i].path), shared_file_ref(sf));
	}
}

/**
 * Look up a possibly shared special file, updating the entry with current
 * file size and modification time.
 *
 * @param path	the URL path on the server (case sensitive, of course)
 *
 * @return the shared file information if there is something shared at path,
 * or NULL if the path is invalid.
 */
shared_file_t *
shared_special(const char *path)
{
	shared_file_t *sf;
	filestat_t file_stat;

	sf = g_hash_table_lookup(special_names, path);

	if (sf == NULL)
		return NULL;

	if (-1 == stat(sf->file_path, &file_stat)) {
		g_warning("can't stat %s: %s", sf->file_path, g_strerror(errno));
		return NULL;
	}

	if (!S_ISREG(file_stat.st_mode)) {
		g_warning("file %s is no longer a plain file", sf->file_path);
		return NULL;
	}

	/*
	 * Update information in case the file changed since the last time
	 * we served it.
	 */

	sf->file_size = file_stat.st_size;
	sf->mtime = file_stat.st_mtime;

	return sf;
}

/**
 * Given a valid index, returns the `struct shared_file' entry describing
 * the shared file bearing that index if found, NULL if not found (invalid
 * index) and SHARE_REBUILDING when we're rebuilding the library.
 */
shared_file_t *
shared_file(guint idx)
{
	/* @return shared file info for index `idx', or NULL if none */

	if (file_table == NULL)			/* Rebuilding the library! */
		return SHARE_REBUILDING;

	if (idx < 1 || idx > files_scanned)
		return NULL;

	return file_table[idx - 1];
}

/**
 * Given a valid index, returns the `struct shared_file' entry describing
 * the shared file bearing that index if found, NULL if not found (invalid
 * index) and SHARE_REBUILDING when we're rebuilding the library.
 */
shared_file_t *
shared_file_sorted(guint idx)
{
	/* @return shared file info for index `idx', or NULL if none */

	if (sorted_file_table == NULL)			/* Rebuilding the library! */
		return SHARE_REBUILDING;

	if (idx < 1 || idx > files_scanned)
		return NULL;

	return sorted_file_table[idx - 1];
}

/**
 * Get index of shared file identified by its name.
 * @return index > 0 if found, 0 if file is not known.
 */
static guint
shared_file_get_index(const char *filename)
{
	guint idx;

	idx = GPOINTER_TO_UINT(g_hash_table_lookup(file_basenames, filename));
	if (idx == 0 || idx == FILENAME_CLASH)
		return 0;

	g_assert(idx >= 1 && idx <= files_scanned);
	return idx;	
}

/**
 * Given a file basename, returns the `struct shared_file' entry describing
 * the shared file bearing that basename, provided it is unique, NULL if
 * we either don't have a unique filename or SHARE_REBUILDING if the library
 * is being rebuilt.
 */
shared_file_t *
shared_file_by_name(const char *filename)
{
	shared_file_t *sf;
	guint idx;

	if (file_table == NULL)
		return SHARE_REBUILDING;

	g_assert(file_basenames);
	idx = shared_file_get_index(filename);
	if (idx > 0) {
		sf = file_table[idx - 1];
		shared_file_check(sf);
		return sf;
	} else {
		return NULL;
	}
}

static void
free_extensions_helper(gpointer key,
	gpointer unused_value, gpointer unused_data)
{
	(void) unused_value;
	(void) unused_data;
	atom_str_free(key);
}
/**
 * Free existing extensions
 */
static void
free_extensions(void)
{
	if (extensions) {
		g_hash_table_foreach(extensions, free_extensions_helper, NULL);
		gm_hash_table_destroy_null(&extensions);
	}
}

/**
 * Get the file extensions to scan.
 */
void
parse_extensions(const char *str)
{
	char **exts = g_strsplit(str, ";", 0);
	char *x, *s;
	guint i;

	free_extensions();
	extensions = g_hash_table_new(ascii_strcase_hash, ascii_strcase_eq);

	for (i = 0; exts[i]; i++) {
		char c;

		s = exts[i];
		while ((c = *s) == '.' || c == '*' || c == '?' || is_ascii_blank(c))
			s++;

		if (c) {

			for (x = strchr(s, '\0'); x-- != s; /* NOTHING */) {
				if ((c = *x) == '*' || c == '?' || is_ascii_blank(c))
					*x = '\0';
				else
					break;
			}

			if (*s && NULL == g_hash_table_lookup(extensions, s)) {
				gconstpointer key = atom_str_get(s);
				gm_hash_table_insert_const(extensions, key, key);
			}
		}
	}

	g_strfreev(exts);
}

/**
 * Release shared dirs.
 */
static void
shared_dirs_free(void)
{
	GSList *sl;

	if (!shared_dirs)
		return;

	for (sl = shared_dirs; sl; sl = g_slist_next(sl)) {
		atom_str_free(sl->data);
	}
	gm_slist_free_null(&shared_dirs);
}

/**
 * Update the property holding the shared directories.
 */
void
shared_dirs_update_prop(void)
{
    GSList *sl;
    str_t *s;

    s = str_new(0);

    for (sl = shared_dirs; sl != NULL; sl = g_slist_next(sl)) {
        str_cat(s, sl->data);
        if (g_slist_next(sl) != NULL)
            str_putc(s, G_SEARCHPATH_SEPARATOR);
    }

    gnet_prop_set_string(PROP_SHARED_DIRS_PATHS, str_2c(s));

    str_destroy(s);
}

/**
 * Parses the given string and updated the internal list of shared dirs.
 * The given string was completely parsed, it returns TRUE, otherwise
 * it returns FALSE.
 */
gboolean
shared_dirs_parse(const char *str)
{
	char **dirs = g_strsplit(str, G_SEARCHPATH_SEPARATOR_S, 0);
    gboolean ret = TRUE;
	guint i;

	/* FIXME: ESCAPING! */

	shared_dirs_free();

	for (i = 0; dirs[i]; i++) {
		if (is_directory(dirs[i]))
			shared_dirs = g_slist_prepend(shared_dirs,
								deconstify_gchar(atom_str_get(dirs[i])));
        else
            ret = FALSE;
	}

	shared_dirs = g_slist_reverse(shared_dirs);
	g_strfreev(dirs);

    return ret;
}

/**
 * Add directory to the list of shared directories.
 */
void
shared_dir_add(const char *pathname)
{
	if (is_directory(pathname)) {
		if (GNET_PROPERTY(share_debug) > 0) {
			g_debug("%s: adding pathname=\"%s\"", G_STRFUNC, pathname);
		}
        shared_dirs = g_slist_append(shared_dirs,
						deconstify_gchar(atom_str_get(pathname)));
	} else {
		if (GNET_PROPERTY(share_debug) > 0) {
			g_debug("%s: NOT adding pathname=\"%s\"", G_STRFUNC, pathname);
		}
	}
    shared_dirs_update_prop();
}

/**
 * Add one more reference to a shared_file_t.
 * @return its argument, for convenience.
 */
shared_file_t *
shared_file_ref(const shared_file_t *sf)
{
	shared_file_t *wsf = deconstify_gpointer(sf);

	wsf->refcnt++;
	return wsf;
}

/**
 * Remove one reference to a shared_file_t, freeing entry if there are
 * no reference left. The pointer itself is nullified.
 */
void
shared_file_unref(shared_file_t **sf_ptr)
{
	g_assert(sf_ptr);

	if (*sf_ptr) {
		shared_file_t *sf = *sf_ptr;

		shared_file_check(sf);
		g_assert(sf->refcnt > 0);

		sf->refcnt--;
		if (0 == sf->refcnt) {
			shared_file_free(&sf);
		}
		*sf_ptr = NULL;
	}
}

/**
 * Is file too big to be shared on Gnutella?
 *
 * Note: The original purpose was to avoid files larger than 2^32-1 bytes.
 *		 Keep it just in case that a platform has an fileoffset_t with more than
 *		 64 bits.
 */
static inline gboolean
too_big_for_gnutella(fileoffset_t size)
{
	g_return_val_if_fail(size >= 0, TRUE);
	return size + (filesize_t)0 > (filesize_t)-1 + (fileoffset_t)0;
}

/**
 * Checks whether it's OK to share the pathname with respect to special
 * characters in the string. As the database stores records line-by-line,
 * newline characters in the filename are not acceptable.
 *
 * @return	If the pathname contains ASCII control characters, TRUE is
 *			returned. Otherwise, the pathname is considered OK and FALSE
 *			is returned.
 */
static gboolean
contains_control_chars(const char *pathname)
{
	const char *s;

	for (s = pathname; *s != '\0'; s++) {
		if (is_ascii_cntrl(*s))
			return TRUE;
	}
	return FALSE;
}

/**
 * Extracts the relative path from a pathname relative to base_dir. If
 * base_dir and pathname do not overlap, NULL is returned. The resulting
 * is converted to UTF-8 NFC and returned as an atom.
 *
 * @param base_dir The base directory.
 * @param pathname A pathname that is relative to "base_dir".
 * @return A string atom holding the relative path or NULL.
 */
static const char *
get_relative_path(const char *base_dir, const char *pathname)
{
	const char *s, *relative_path = NULL;

	s = is_strprefix(pathname, base_dir);
	if (s) {
		s = skip_dir_separators(s);
		if ('\0' != s[0]) {
			char *nfc_str;

			nfc_str = filename_to_utf8_normalized(s, UNI_NORM_NETWORK);
			normalize_dir_separators(nfc_str);
			relative_path = atom_str_get(nfc_str);
			G_FREE_NULL(nfc_str);
		}
	}
	return relative_path;
}

/**
 * Verify that a file extension is valid for sharing
 *
 * @param filename  The name of the file to check.
 * @return TRUE if the file should be shared, FALSE if not.
 */
static gboolean
shared_file_valid_extension(const char *filename)
{
	const char *filename_ext;

	if (!extensions)
		return FALSE;

	if (
		1 == g_hash_table_size(extensions) &&
		g_hash_table_lookup(extensions, "--all--")
    ) {
		/*
		 * An extension "--all--" matches all files, even those that don't
		 * have any extension. [Original patch by Zygo Blaxell].
		 */
		return TRUE;
	}

	filename_ext = strrchr(filename, '.');
	if (filename_ext) {
		/*
		 * Filenames without any extension are not shared, unless
		 * "--all--" is used.
		 */	

		filename_ext++;	/* skip the dot */

		/* 
		 * Match the file extension (if any) against the extensions list.
		 * All valid extensions start with '.'.  Matching is case-insensitive
		 */

		if (g_hash_table_lookup(extensions, filename_ext)) {
			return TRUE;
		}
	}

	return FALSE;
}

/**
 * @param relative_path The relative path of the file or NULL.
 * @param pathname The absolute pathname of the file.
 * @param sb A "stat buffer" that was initialized with stat().
 *
 * return On success a shared_file_t for the file is returned. Otherwise,
 *		  NULL is returned.
 */
static shared_file_t * 
share_scan_add_file(const char *relative_path,
	const char *pathname, const filestat_t *sb)
{
	shared_file_t *sf;
	const char *name;

	g_assert(is_absolute_path(pathname));
	g_assert(sb);
	g_return_val_if_fail(S_ISREG(sb->st_mode), NULL);

	if (0 == sb->st_size) {
		if (GNET_PROPERTY(share_debug) > 5)
			g_warning("not sharing empty file: \"%s\"", pathname);
		return NULL;
	}

	if (too_big_for_gnutella(sb->st_size)) {
		g_warning("file is too big to be shared: \"%s\"", pathname);
		return NULL;
	}

	if (contains_control_chars(pathname)) {
		g_warning("not sharing filename with control characters: "
				"\"%s\"", pathname);
		return NULL;
	}

	if (!shared_file_valid_extension(pathname))
		return NULL;

	name = filepath_basename(pathname);

	if (GNET_PROPERTY(share_debug) > 5)
		g_debug("%s: pathname=\"%s\"", G_STRFUNC, pathname);

	sf = shared_file_alloc();
	sf->file_path = atom_str_get(pathname);
	sf->relative_path = relative_path ? atom_str_get(relative_path) : NULL;
	sf->file_size = sb->st_size;
	sf->mtime = sb->st_mtime;

	if (shared_file_set_names(sf, name)) {
		shared_file_free(&sf);
		return NULL;
	}

	if (spam_check_filename_size(sf->name_nfc, sf->file_size)) {
		g_warning("file \"%s\" is listed as spam (Name)", sf->name_nfc);
		shared_file_free(&sf);
		return NULL;
	}

	sf->mime_type = mime_type_from_filename(sf->name_nfc);

	if (!sha1_is_cached(sf)) {
		int ret;

		/*
		 * In the "tmp" directory, don't share files that have a trailer.
		 * It's probably a file being downloaded, and which is not complete
		 * yet.  This check is necessary in case they chose to share their
		 * downloading directory...
		 */

		ret = file_info_has_trailer(pathname);
		switch (ret) {
		case 1:
			/*
			 * It's probably a file being downloaded, and which
			 * is not complete yet. This check is necessary in
			 * case they choose to share their downloading
			 * directory...
			 */
			g_warning("will not share partial file \"%s\"", pathname);
			/* FALL THROUGH */
		case -1:
			shared_file_free(&sf);
			return NULL;
		}
	}
	
	/*
	 * NOTE: An `else'-clause here would be if the file WAS found in the
	 * sha1_cache.  Good place to set the SHA1 in "sf".
	 */

	return sf;
}

/**
 * Check whether the given directory is one which should never be shared.
 * This is not meant to be exhaustive but test for common configuration
 * mistakes.
 */
static gboolean
directory_is_unshareable(const char *dir)
{
	g_assert(dir);

	/* Explicitly checking is_same_file() for TRUE to ignore errors (-1)
	 * probably caused by non-existing files or missing permission.
	 */
	if (TRUE == is_same_file(dir, "/")) {
		g_warning("refusing to share root directory: %s", dir);
		return TRUE;
	}

	if (TRUE == is_same_file(dir, settings_home_dir())) {
		g_warning("refusing to share home directory: %s", dir);
		return TRUE;
	}

	if (TRUE == is_same_file(dir, settings_config_dir())) {
		g_warning("refusing to share directory for configuration data: %s",
			dir);
		return TRUE;
	}

	if (
		!is_null_or_empty(GNET_PROPERTY(save_file_path)) &&
		TRUE == is_same_file(dir, GNET_PROPERTY(save_file_path))
	) {
		g_warning("refusing to share directory for incomplete files: %s", dir);
		return TRUE;
	}

	if (TRUE == is_same_file(dir, GNET_PROPERTY(bad_file_path))) {
		g_warning("refusing to share directory for corrupted files: %s", dir);
		return TRUE;
	}

	return FALSE;	/* No objection */
}

enum recursive_scan_magic { RECURSIVE_SCAN_MAGIC = 0x16926d87U };

struct recursive_scan {
	enum recursive_scan_magic magic;	/**< Magic number. */
	struct bgtask *task;
	DIR *directory;
	const char *base_dir;		/* string atom */
	const char *current_dir;	/* string atom */
	const char *relative_path;	/* string atom */
	slist_t *base_dirs;			/* list of string atoms */
	slist_t *sub_dirs;			/* list of g_malloc()ed strings */
	slist_t *shared_files;		/* list of struct shared_file */
	slist_t *partial_files;		/* list of struct shared_file */
	slist_iter_t *iter;			/* list iterator */
	GHashTable *words;			/* records words making up filenames, for QRP */
	GHashTable *basenames;		/* known file basenames */
	GSList *shared;				/* the new shared_files variable */
	shared_file_t **files;		/* the new file_table, sorted by mtime */
	shared_file_t **sorted;		/* the new sorted_file_table, sorted by name */
	search_table_t *search_tb;	/* the new search table */
	search_table_t *partial_tb;	/* the new partial table */
	guint64 files_scanned;		/* amount of files shared in the library */
	guint64 bytes_scanned;		/* size of the library */
	int idx;					/* iterating index */
	int ticks;					/* ticks used */
};

static inline void
recursive_scan_check(const struct recursive_scan * const ctx)
{
	g_assert(ctx);
	g_assert(RECURSIVE_SCAN_MAGIC == ctx->magic);
	g_assert(ctx->base_dirs != NULL);
	g_assert(ctx->sub_dirs != NULL);
	g_assert(ctx->shared_files != NULL);
	g_assert(ctx->partial_files != NULL);
}

static struct recursive_scan *
recursive_scan_new(const GSList *base_dirs)
{
	struct recursive_scan *ctx;
	const GSList *iter;

	WALLOC0(ctx);
	ctx->magic = RECURSIVE_SCAN_MAGIC;
	ctx->base_dirs = slist_new();
	ctx->sub_dirs = slist_new();
	ctx->shared_files = slist_new();
	ctx->partial_files = slist_new();
	ctx->words = g_hash_table_new(g_str_hash, g_str_equal);
	ctx->basenames = g_hash_table_new(g_str_hash, g_str_equal);
	for (iter = base_dirs; NULL != iter; iter = g_slist_next(iter)) {
		const char *dir = atom_str_get(iter->data);
		slist_append(ctx->base_dirs, deconstify_gchar(dir));
	}
	return ctx;
}

static void
recursive_scan_closedir(struct recursive_scan *ctx)
{
	recursive_scan_check(ctx);

	if (GNET_PROPERTY(share_debug) > 6 && ctx->current_dir != NULL)
		g_debug("SHARE leaving directory \"%s\"", ctx->current_dir);

	atom_str_free_null(&ctx->relative_path);
	atom_str_free_null(&ctx->current_dir);
	if (ctx->directory) {
		closedir(ctx->directory);
		ctx->directory = NULL;
	}
}

static void recursive_sf_unref(gpointer o)
{
	shared_file_t *sf = o;

	shared_file_check(sf);
	shared_file_unref(&sf);
}

/**
 * Encapsulation of hfree() in case TRACK_MALLOC is defined and hfree() is
 * really a macro, not a function.
 */
static void
do_hfree(void *p)
{
	hfree(p);
}

static void
recursive_scan_free(struct recursive_scan **ctx_ptr)
{
	g_assert(ctx_ptr);

	if (*ctx_ptr) {
		struct recursive_scan *ctx = *ctx_ptr;
		GSList *sl;

		recursive_scan_check(ctx);

		if (ctx->task) {
			bg_task_cancel(ctx->task);
			g_assert(NULL == ctx->task);
		}

		recursive_scan_closedir(ctx);

		slist_free_all(&ctx->base_dirs, (slist_destroy_cb) atom_str_free);
		slist_free_all(&ctx->sub_dirs, do_hfree);
		slist_free_all(&ctx->shared_files, recursive_sf_unref);
		slist_free_all(&ctx->partial_files, recursive_sf_unref);
		slist_iter_free(&ctx->iter);

		gm_hash_table_destroy_null(&ctx->basenames);
		st_free(&ctx->search_tb);
		st_free(&ctx->partial_tb);
		atom_str_free_null(&ctx->base_dir);
		qrp_dispose_words(&ctx->words);

		HFREE_NULL(ctx->files);
		HFREE_NULL(ctx->sorted);

		for (sl = ctx->shared; sl; sl = g_slist_next(sl)) {
			shared_file_t *sf = sl->data;

			shared_file_check(sf);
			shared_file_unref(&sf);
		}
		gm_slist_free_null(&ctx->shared);

		ctx->magic = 0;
		WFREE(ctx);
		*ctx_ptr = NULL;
	}
}

static void
recursive_scan_context_free(void *data)
{
	struct recursive_scan *ctx = data;
	
	recursive_scan_check(ctx);
	/* If we're called, the task is being terminated */

	/*
	 * There's nothing to free here.  We're managing the overall context
	 * ourselves and this overall context is given as the task's context.
	 * We just need to record the fact that the task has been terminated
	 * here so recursive_scan_free() does not try to cancel it.
	 */

	ctx->task = NULL;
}

/**
 * Free up memory used by the shared library.
 */
static void
share_free(void)
{
	GSList *sl;

	st_free(&search_table);
	gm_hash_table_destroy_null(&file_basenames);

	for (sl = shared_files; sl; sl = g_slist_next(sl)) {
		shared_file_t *sf = sl->data;

		shared_file_check(sf);
		shared_file_unref(&sf);
	}
	gm_slist_free_null(&shared_files);

	HFREE_NULL(file_table);
	HFREE_NULL(sorted_file_table);
}

/**
 * Sort function - shared files by ascending mtime (oldest first). 
 */
static int
shared_file_sort_by_mtime(const void *f1, const void *f2)
{
	const shared_file_t * const *sfp1 = f1, * const *sfp2 = f2;
	const shared_file_t *sf1 = *sfp1, *sf2 = *sfp2;
	time_t t1, t2;

	/* We don't use shared_file_check() here because it would be
	 * the dominating factor for the sorting time. */
	g_assert(SHARED_FILE_MAGIC == sf1->magic);
	g_assert(SHARED_FILE_MAGIC == sf2->magic);

	t1 = sf1->mtime;
	t2 = sf2->mtime;
	return CMP(t1, t2);
}

static inline int
cmp_strings(const char *a, const char *b)
{
	if (a && b) {
		return a == b ? 0 : strcmp(a, b);
	} else {
		return a ? 1 : (b ? -1 : 0);
	}
}

/**
 * Sort function - shared files by name lexicographic order.
 */
static int
shared_file_sort_by_name(const void *f1, const void *f2)
{
	const shared_file_t * const *sfp1 = f1, * const *sfp2 = f2;
	const shared_file_t *sf1 = *sfp1, *sf2 = *sfp2;
	int ret;

	/* We don't use shared_file_check() here because it would be
	 * the dominating factor for the sorting time. */
	g_assert(SHARED_FILE_MAGIC == sf1->magic);
	g_assert(SHARED_FILE_MAGIC == sf2->magic);

	if (GNET_PROPERTY(search_results_expose_relative_paths)) {
		ret = cmp_strings(sf1->relative_path, sf2->relative_path);
	} else {
		ret = strcmp(sf1->file_path, sf2->file_path);
	}
	return 0 != ret ? ret : strcmp(sf1->name_nfc, sf2->name_nfc);
}

static void
recursive_scan_opendir(struct recursive_scan *ctx, const char * const dir)
{
	recursive_scan_check(ctx);
	g_assert(NULL == ctx->directory);
	g_assert(NULL == ctx->relative_path);
	g_assert(NULL == ctx->current_dir);

	g_return_if_fail('\0' != dir[0]);
	g_return_if_fail(is_absolute_path(ctx->base_dir));
	g_return_if_fail(is_absolute_path(dir));

	if (directory_is_unshareable(dir))
		return;

	/**
	 * FIXME: On Windows FindFirstFile/FindNextFile/FindClose
	 *		  must be used to get the Unicode filenames.		
	 */
	if (!(ctx->directory = opendir(dir))) {
		g_warning("can't open directory %s: %s", dir, g_strerror(errno));
		return;
	}

	/* Get relative path if required */
	if (GNET_PROPERTY(search_results_expose_relative_paths)) {
		ctx->relative_path = get_relative_path(ctx->base_dir, dir);
	} else {
		ctx->relative_path = NULL;
	}
	ctx->current_dir = atom_str_get(dir);

	if (GNET_PROPERTY(share_debug) > 5)
		g_debug("SHARE scanning directory \"%s\"", ctx->current_dir);
}

static void
recursive_scan_readdir(struct recursive_scan *ctx)
{
	char *fullpath = NULL;
	struct dirent *dir_entry;

	recursive_scan_check(ctx);
	g_assert(ctx->directory);

	dir_entry = readdir(ctx->directory);
	if (dir_entry) {
		const char *filename = dir_entry_filename(dir_entry);
		filestat_t sb;

		if (GNET_PROPERTY(share_debug) > 19)
			g_debug("SHARE considering entry \"%s\"", filename);

		if ('.' == filename[0]) {
			/* Hidden file, or "." or ".." */
			goto finish;
		}

		sb.st_mode = dir_entry_mode(dir_entry);
		switch (sb.st_mode) {
		case 0:
		case S_IFREG:
		case S_IFDIR:
		case S_IFLNK:
			break;
		default:
			if (GNET_PROPERTY(share_debug)) {
				g_warning("skipping file of unknown type \"%s\" in \"%s\"",
					ctx->current_dir, filename);
			}
			goto finish;
		}

		if (
			S_ISLNK(sb.st_mode) &&
			GNET_PROPERTY(scan_ignore_symlink_dirs) &&
			GNET_PROPERTY(scan_ignore_symlink_regfiles)
		) {
			if (GNET_PROPERTY(share_debug) > 15) {
				g_debug("SHARE to-be-ignored symlink, discarding \"%s\"",
					filename);
			}
			goto finish;
		}

		if (
			S_ISREG(sb.st_mode) &&
			!shared_file_valid_extension(filename)
		) {
			if (GNET_PROPERTY(share_debug) > 15) {
				g_debug("SHARE unshared extension, discarding \"%s\"",
					filename);
			}
			goto finish;
		}

		ctx->ticks += 10;	/* Heavier work */

		fullpath = make_pathname(ctx->current_dir, filename);
		if (S_ISREG(sb.st_mode) || S_ISDIR(sb.st_mode)) {
			if (stat(fullpath, &sb)) {
				g_warning("stat() failed %s: %s", fullpath, g_strerror(errno));
				goto finish;
			}
		} else if (!S_ISLNK(sb.st_mode)) {
			if (lstat(fullpath, &sb)) {
				g_warning("lstat() failed %s: %s", fullpath, g_strerror(errno));
				goto finish;
			}

			if (
				S_ISLNK(sb.st_mode) &&
				GNET_PROPERTY(scan_ignore_symlink_dirs) &&
				GNET_PROPERTY(scan_ignore_symlink_regfiles)
			) {
				/*
				 * We check this again because dir_entry_mode() does not
				 * work everywhere.
				 */
				if (GNET_PROPERTY(share_debug) > 15) {
					g_debug("SHARE to-be-ignored symlink, discarding \"%s\"",
						filename);
				}
				goto finish;
			}
		}

		/* Get info on the symlinked file */
		if (S_ISLNK(sb.st_mode)) {
			if (stat(fullpath, &sb)) {
				g_warning("broken symlink %s: %s", fullpath, g_strerror(errno));
				goto finish;
			}
			
			/*
			 * For symlinks, we check whether we are supposed to process
			 * symlinks for that type of entry, then either proceed or skip the
			 * entry.
			 */

			if (
				S_ISDIR(sb.st_mode) &&
				GNET_PROPERTY(scan_ignore_symlink_dirs)
			) {
				if (GNET_PROPERTY(share_debug) > 15)
					g_debug("SHARE discarding symlink dir \"%s\"", filename);
				goto finish;
			}
			if (
				S_ISREG(sb.st_mode) &&
				GNET_PROPERTY(scan_ignore_symlink_regfiles)
			) {
				if (GNET_PROPERTY(share_debug) > 15)
					g_debug("SHARE discarding symlink file \"%s\"", filename);
				goto finish;
			}
		}
		
		if (S_ISDIR(sb.st_mode)) {
			/* If a directory, add to list for later processing */
			slist_prepend(ctx->sub_dirs, fullpath);
			fullpath = NULL;
		} else if (S_ISREG(sb.st_mode)) {
			shared_file_t *sf;

			if (GNET_PROPERTY(share_debug) > 10)
				g_debug("SHARE adding file \"%s\"", filename);

			sf = share_scan_add_file(ctx->relative_path, fullpath, &sb);
			if (sf) {
				slist_append(ctx->shared_files, shared_file_ref(sf));
			}
		}
	} else {
		recursive_scan_closedir(ctx);
	}

finish:
	HFREE_NULL(fullpath);
	dir_entry_filename(NULL);	/* release memory */
}

/**
 * @return TRUE if finished.
 */
static gboolean 
recursive_scan_next_dir(struct recursive_scan *ctx)
{
	recursive_scan_check(ctx);

	if (ctx->directory) {
		recursive_scan_readdir(ctx);
		return FALSE;
	} else if (slist_length(ctx->sub_dirs) > 0) {
		char *dir;
	   
		dir = slist_shift(ctx->sub_dirs);
		recursive_scan_opendir(ctx, dir);
		HFREE_NULL(dir);
		return FALSE;
	} else if (slist_length(ctx->base_dirs) > 0) {
		atom_str_free_null(&ctx->base_dir);
		ctx->base_dir = slist_shift(ctx->base_dirs);
		recursive_scan_opendir(ctx, ctx->base_dir);
		return FALSE;
	} else {
		atom_str_free_null(&ctx->base_dir);
		return TRUE;
	}
}

static bgret_t
recursive_scan_step_compute(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);

	ctx->ticks = 0;
	do {
		if (recursive_scan_next_dir(ctx)) {
			bg_task_ticks_used(bt, ctx->ticks);
			return BGR_NEXT;
		}
		ctx->ticks++;
	} while (ctx->ticks < ticks);

	return BGR_MORE;
}

static bgret_t
recursive_scan_step_compute_done(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);
	(void) ticks;

	g_assert(NULL == ctx->shared);
	g_assert(NULL == ctx->search_tb);

	ctx->files_scanned = slist_length(ctx->shared_files);
	ctx->bytes_scanned = 0;
	ctx->search_tb = st_create();

	bg_task_ticks_used(bt, 0);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_build_search_table(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);

	ctx->ticks = 0;

	while (slist_length(ctx->shared_files) > 0) {
		const shared_file_t *sf;

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;

		sf = slist_shift(ctx->shared_files);
		shared_file_check(sf);
		g_assert(!shared_file_is_partial(sf));
		ctx->bytes_scanned += sf->file_size;
		st_insert_item(ctx->search_tb, sf->name_canonic, sf);
		ctx->shared = gm_slist_prepend_const(ctx->shared, sf);
		upload_stats_enforce_local_filename(sf);
	}

	/* Compact the search table */
	st_compact(ctx->search_tb);
	ctx->ticks += 5;

	bg_task_ticks_used(bt, ctx->ticks);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_build_file_table(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	GSList *sl;
	int i = 0;

	recursive_scan_check(ctx);
	g_assert(NULL == ctx->files);

	(void) ticks;

	if (0 == ctx->files_scanned)
		goto next;

	/*
	 * In order to quickly locate files based on indicies, build a table
	 * of all shared files.  This table is only accessible via shared_file().
	 * NB: file indicies start at 1, but indexing in table starts at 0.
	 *		--RAM, 08/10/2001
	 */

	ctx->files = halloc0(ctx->files_scanned * sizeof ctx->files[0]);

	for (i = 0, sl = ctx->shared; sl; sl = g_slist_next(sl)) {
		shared_file_t *sf = sl->data;

		shared_file_check(sf);
		g_assert(!(SHARE_F_INDEXED & sf->flags));
		g_assert(UNSIGNED(i) < ctx->files_scanned);
		ctx->files[i++] = sf;
	}

	/* Sort file list by modification time to get a relatively stable index */
	qsort(ctx->files, ctx->files_scanned, sizeof ctx->files[0],
		shared_file_sort_by_mtime);

next:
	bg_task_ticks_used(bt, i / 10);
	ctx->idx = 0;				/* Prepares next step */

	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_build_basenames(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);

	ctx->ticks = 0;

	while (UNSIGNED(ctx->idx) < ctx->files_scanned) {
		shared_file_t *sf;
		guint val;
		int i = ctx->idx++;

	   	sf = ctx->files[i];
		shared_file_check(sf);

		/*
		 * Set file_index based on new sort order.
		 *
		 * We don't set SHARE_F_INDEXED yet, this will be done only when
		 * we're ready to install the new data structures we're still building.
		 */

		sf->file_index = i + 1;

		/*
		 * In order to transparently handle files requested with the wrong
		 * indices, for older servents that would not know how to handle a
		 * return code of "301 Moved" with a Location header, we keep track
		 * of individual basenames of files, recording the index of each file.
		 * As soon as there is a clash, we revoke the entry by storing
		 * FILENAME_CLASH instead, which cannot be a valid index.
		 *		--RAM, 06/06/2002
		 */

		val = pointer_to_uint(
				g_hash_table_lookup(ctx->basenames, sf->name_nfc));

		/*
		 * The following works because 0 cannot be a valid file index.
		 */

		val = (val != 0) ? FILENAME_CLASH : sf->file_index;
		g_hash_table_insert(ctx->basenames, deconstify_gchar(sf->name_nfc),
			GUINT_TO_POINTER(val));

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;
	}

	bg_task_ticks_used(bt, ctx->ticks);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_update_scan_timing(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	time_delta_t elapsed;

	recursive_scan_check(ctx);
	(void) ticks;

	elapsed = delta_time(tm_time_exact(),
				GNET_PROPERTY(library_rescan_started));
	elapsed = MAX(0, elapsed);
	gnet_prop_set_timestamp_val(PROP_LIBRARY_RESCAN_FINISHED, tm_time());
	gnet_prop_set_guint32_val(PROP_LIBRARY_RESCAN_DURATION, elapsed);

	bg_task_ticks_used(bt, 0);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_build_sorted_table(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	int i = 0;

	recursive_scan_check(ctx);
	g_assert(NULL == ctx->sorted);

	(void) ticks;

	if (0 == ctx->files_scanned)
		goto next;

	ctx->sorted = hcopy(ctx->files, ctx->files_scanned * sizeof ctx->files[0]);

	qsort(ctx->sorted, ctx->files_scanned, sizeof ctx->sorted[0],
		shared_file_sort_by_name);

	/*
	 * Set the sort index used for sorted file listings
	 *
	 * Note that we're not setting SHARE_F_INDEXED yet because we don't know
	 * whether we're going to install the data structures we're building.
	 */

	for (i = 0; UNSIGNED(i) < ctx->files_scanned; i++) {
		shared_file_t *sf;

	   	sf = ctx->sorted[i];
		shared_file_check(sf);
		sf->sort_index = i + 1;
	}

next:
	bg_task_ticks_used(bt, i / 10);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_install_shared(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	size_t i;

	recursive_scan_check(ctx);
	g_assert(ctx->search_tb != NULL);

	(void) ticks;

	/*
	 * This step is "atomic" in that it cannot be interrupted by the background
	 * task scheduler.
	 *
	 * We're installing the shared library data structures we've been building
	 * in the previous steps, discarding the old ones.
	 */

	if (GNET_PROPERTY(share_debug) > 1) {
		int count = st_count(ctx->search_tb);
		g_debug("SHARE installing new search table (%d item%s)",
			count, 1 == count ? "" : "s");
	}

	share_free();

	search_table = ctx->search_tb;
	file_basenames = ctx->basenames;
	shared_files = ctx->shared;
	file_table = ctx->files;
	sorted_file_table = ctx->sorted;
	files_scanned = ctx->files_scanned;
	bytes_scanned = ctx->bytes_scanned;

	/*
	 * Now that we installed the shared files, we can mark the entries as
	 * being indexed and basenamed.
	 */

	for (i = 0; i < files_scanned; i++) {
		shared_file_t *sf = file_table[i];

		shared_file_check(sf);
		sf->flags |= SHARE_F_INDEXED | SHARE_F_BASENAME;
	}

	/*
	 * Reset these contextual variables, they are now held by the global ones.
	 */

	ctx->search_tb = NULL;
	ctx->basenames = NULL;
	ctx->shared = NULL;
	ctx->files = NULL;
	ctx->sorted = NULL;

	gcu_gui_update_files_scanned();		/* Final view */
	gnet_prop_set_boolean_val(PROP_LIBRARY_REBUILDING, FALSE);

	/*
	 * The next step is going to request the SHA1 of all the library files,
	 * which will fill again the known SHA1 cache.
	 */

	reinit_sha1_table();

	bg_task_ticks_used(bt, ctx->files_scanned / 10);
	ctx->idx = 0;		/* Prepare next step */
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_request_sha1(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);

	/*
	 * The new library has been installed, we must now access the global
	 * variables.
	 */

	g_assert(NULL == ctx->files);		/* Indicates installation was done */

	ctx->ticks = 0;

	while (UNSIGNED(ctx->idx) < files_scanned) {
		shared_file_t *sf;
		int i = ctx->idx++;

	   	sf = file_table[i];
		shared_file_check(sf);

		/*
		 * We must not change the file index after request_sha1() since this
		 * can synchronously call routines to set the SHA1 if it's known
		 * already.
		 */

		request_sha1(sf);

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;
	}

	share_rebuilding = FALSE;	/* Done rebuilding the SHA1 table */
	bg_task_ticks_used(bt, ctx->ticks);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_load_partials(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);
	(void) ticks;

	if (share_can_answer_partials()) {
		hash_list_iter_t *iter = hash_list_iterator(partial_files);

		while (hash_list_iter_has_next(iter)) {
			const shared_file_t *sf = hash_list_iter_next(iter);
			slist_append(ctx->partial_files, shared_file_ref(sf));
		}

		hash_list_iter_release(&iter);
	}

	bg_task_ticks_used(bt, 0);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_build_partial_table(struct bgtask *bt,
	void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);

	ctx->ticks = 0;

	if (NULL == ctx->iter) {
		ctx->iter = slist_iter_before_head(ctx->partial_files);
		g_assert(NULL == ctx->partial_tb);
		ctx->partial_tb = st_create();
	}

	if (!share_can_answer_partials())
		goto next;

	while (slist_iter_has_next(ctx->iter)) {
		const shared_file_t *sf = slist_iter_next(ctx->iter);

		shared_file_check(sf);
		g_assert(shared_file_is_partial(sf));
		st_insert_item(ctx->partial_tb, sf->name_canonic, sf);

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;
	}

	/* Compact the search table */
	st_compact(ctx->partial_tb);
	ctx->ticks += 5;

next:
	slist_iter_free(&ctx->iter);

	bg_task_ticks_used(bt, ctx->ticks);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_install_partials(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);
	g_assert(ctx->partial_tb != NULL);

	(void) ticks;

	/*
	 * Install the new partial table, regardless of whether they allow queries
	 * to return partials: if they don't, the table is empty anyway.
	 */

	if (GNET_PROPERTY(share_debug) > 1) {
		int count = st_count(ctx->partial_tb);
		g_debug("SHARE installing new partial table (%d item%s)",
			count, 1 == count ? "" : "s");
	}

	st_free(&partial_table);
	partial_table = ctx->partial_tb;
	ctx->partial_tb = NULL;

	bg_task_ticks_used(bt, 0);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_prepare_qrp(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;

	recursive_scan_check(ctx);
	(void) ticks;

	gnet_prop_set_timestamp_val(PROP_QRP_INDEXING_STARTED, tm_time_exact());
	qrp_prepare_computation();
	ctx->idx = 0;

	bg_task_ticks_used(bt, 0);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_update_qrp(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	shared_file_t *sf;

	recursive_scan_check(ctx);

	ctx->ticks = 0;

	for (/* empty */; UNSIGNED(ctx->idx) < files_scanned; ctx->idx++) {
		sf = sorted_file_table[ctx->idx];

		if (!sf)
			continue;

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;

		shared_file_check(sf);
		qrp_add_file(sf, ctx->words);
	}

	while (NULL != (sf = slist_shift(ctx->partial_files))) {
		shared_file_check(sf);

		/*
		 * Assign a unique file index to each partial file, but don't flag
		 * the partial file with SHARE_F_INDEXED as we don't map the index
		 * to the partial file (we only allow retrieval of partials by SHA1).
		 *
		 * The file index is required to send proper information in query hits
		 * when inserting partial files.
		 */

		sf->file_index = files_scanned + (hash_list_length(partial_files) -
			slist_length(ctx->partial_files));

		qrp_add_file(sf, ctx->words);
		shared_file_unref(&sf);

		if (ctx->ticks++ >= ticks)
			return BGR_MORE;
	}

	bg_task_ticks_used(bt, ctx->ticks);
	return BGR_NEXT;
}

static bgret_t
recursive_scan_step_finalize(struct bgtask *bt, void *data, int ticks)
{
	struct recursive_scan *ctx = data;
	time_delta_t elapsed;

	recursive_scan_check(ctx);
	(void) bt;
	(void) ticks;

	qrp_finalize_computation(ctx->words);
	ctx->words = NULL;		/* Gave pointer, QRP computation will free it */

	elapsed = delta_time(tm_time(), GNET_PROPERTY(qrp_indexing_started));
	elapsed = MAX(0, elapsed);
	gnet_prop_set_guint32_val(PROP_QRP_INDEXING_DURATION, elapsed);

	return BGR_DONE;
}

static void
recursive_scan_create_task(struct recursive_scan *ctx)
{
	recursive_scan_check(ctx);

	if (NULL == ctx->task) {
		static const bgstep_cb_t steps[] = {
			recursive_scan_step_compute,
			recursive_scan_step_compute_done,
			recursive_scan_step_build_search_table,
			recursive_scan_step_build_file_table,
			recursive_scan_step_build_basenames,
			recursive_scan_step_update_scan_timing,
			recursive_scan_step_build_sorted_table,
			recursive_scan_step_install_shared,
			recursive_scan_step_request_sha1,
			recursive_scan_step_load_partials,
			recursive_scan_step_build_partial_table,
			recursive_scan_step_install_partials,
			recursive_scan_step_prepare_qrp,
			recursive_scan_step_update_qrp,
			recursive_scan_step_finalize,
		};

		ctx->task = bg_task_create("recursive scan",
							steps, G_N_ELEMENTS(steps),
			  				ctx, recursive_scan_context_free,
							NULL, NULL);
	}
}

static void
share_update_qrp_create_task(struct recursive_scan *ctx)
{
	recursive_scan_check(ctx);

	if (NULL == ctx->task) {
		static const bgstep_cb_t steps[] = {
			recursive_scan_step_load_partials,
			recursive_scan_step_build_partial_table,
			recursive_scan_step_install_partials,
			recursive_scan_step_prepare_qrp,
			recursive_scan_step_update_qrp,
			recursive_scan_step_finalize,
		};

		ctx->task = bg_task_create("QRP update",
							steps, G_N_ELEMENTS(steps),
			  				ctx, recursive_scan_context_free,
							NULL, NULL);
	}
}

/**
 * Perform scanning of the shared directories to build up the list of
 * shared files.
 */
void
share_scan(void)
{
	recursive_scan_free(&recursive_scan_context);
	recursive_scan_context = recursive_scan_new(shared_dirs);
	share_rebuilding = TRUE;
	gnet_prop_set_boolean_val(PROP_LIBRARY_REBUILDING, TRUE);
	gnet_prop_set_timestamp_val(PROP_LIBRARY_RESCAN_STARTED, tm_time_exact());
	recursive_scan_create_task(recursive_scan_context);
}

/**
 * Update the QRP table, including both our shared library and our partials.
 *
 * @return whether QRP update task was launched.
 */
static gboolean
share_update_qrp(void)
{
	/*
	 * If we're already rebuilding something, do nothing for now
	 * We'll be called regularily until we can proceed.
	 */

	if (recursive_scan_context != NULL && recursive_scan_context->task != NULL)
		return FALSE;

	recursive_scan_free(&recursive_scan_context);
	recursive_scan_context = recursive_scan_new(NULL);
	share_update_qrp_create_task(recursive_scan_context);

	return TRUE;
}

/**
 * Hash table iterator callback to free the value.
 */
static void
special_free_kv(gpointer unused_key, gpointer val, gpointer unused_udata)
{
	shared_file_t *sf = val;

	(void) unused_key;
	(void) unused_udata;

	g_assert(SHARE_F_SPECIAL & sf->flags);
	shared_file_unref(&sf);
}

/**
 * Get rid of the special file descriptions, if any.
 */
static void
share_special_close(void)
{
	g_hash_table_foreach(special_names, special_free_kv, NULL);
	gm_hash_table_destroy_null(&special_names);
}

/**
 * Shutdown cleanup.
 */
G_GNUC_COLD void
share_close(void)
{
	recursive_scan_free(&recursive_scan_context);
	share_special_close();
	free_extensions();
	share_free();
	shared_dirs_free();
	huge_close();
	qrp_close();
	oob_proxy_close();
	oob_close();
	qhit_close();
	st_free(&partial_table);
	gm_hash_table_destroy_null(&share_media_types);
	hash_list_free(&partial_files);
	st_free(&partial_table);
	cq_cancel(&share_qrp_rebuild_ev);
}

/*
 * SHA1 digest processing
 */

/**
 * Set the SHA1 hash of a given shared_file. Take care of updating the
 * sha1_to_share structure. This function is called from inside the bowels of
 * huge.c when it knows what the hash associated to a file is.
 */
void
shared_file_set_sha1(shared_file_t *sf, const struct sha1 *sha1)
{
	shared_file_check(sf);
	g_assert(!shared_file_is_partial(sf));	/* Cannot be a partial file */

	sf->flags &= ~(SHARE_F_RECOMPUTING | SHARE_F_HAS_DIGEST);
	sf->flags |= sha1 ? SHARE_F_HAS_DIGEST : 0;

	if (sf->sha1) {
		shared_file_t *current;
		gpointer key;

		key = deconstify_gpointer(sf->sha1);
		current = g_tree_lookup(sha1_to_share, key);
		if (current) {
			shared_file_check(current);
			g_assert(SHARE_F_INDEXED & current->flags);

			if (sf == current) {
				g_tree_remove(sha1_to_share, key);
			}
		}
	}

	atom_sha1_change(&sf->sha1, sha1);

	/*
	 * If the file is no longer in the index table, it must not be
	 * put into the tree again. This might happen if a SHA-1 calculation
	 * from a previous rescan finishes after newly initiated rescan.
	 */

	if ((SHARE_F_INDEXED & sf->flags) && sf->sha1) {
		shared_file_t *current;
		gpointer key;

		key = deconstify_gpointer(sf->sha1);
		current = g_tree_lookup(sha1_to_share, key);
		if (current) {
			shared_file_check(current);
			g_assert(SHARE_F_INDEXED & current->flags);
			
			/*
			 * There can be multiple shared files with the same SHA-1.
			 * Only the first found is inserted into the tree.
			 */
			if (GNET_PROPERTY(share_debug) > 0) {
				g_debug("\"%s\" is a duplicate of \"%s\"",
					shared_file_path(sf),
					shared_file_path(current));
			}
		} else {
			/*
			 * New SHA-1 known for this file entry.
			 * Record in the set of shared SHA-1s and publish to the DHT.
			 */
		
			g_tree_insert(sha1_to_share, deconstify_gpointer(sf->sha1), sf);
			publisher_add(sf->sha1);
		}
	}
}

void
shared_file_set_tth(shared_file_t *sf, const struct tth *tth)
{
	shared_file_check(sf);

	g_assert(!shared_file_is_partial(sf));	/* Cannot be a partial file */

	atom_tth_change(&sf->tth, tth);
}

void
shared_file_set_modification_time(shared_file_t *sf, time_t mtime)
{
	shared_file_check(sf);
	sf->mtime = mtime;
}

/**
 * Predicate returning TRUE if the SHA1 hash is available for a given
 * shared_file, FALSE otherwise.
 *
 * Use sha1_hash_is_uptodate() to check for availability and accurateness.
 */
gboolean
sha1_hash_available(const shared_file_t *sf)
{
	shared_file_check(sf);
	return SHARE_F_HAS_DIGEST ==
		(sf->flags & (SHARE_F_HAS_DIGEST | SHARE_F_RECOMPUTING));
}

/**
 * Predicate returning TRUE if the SHA1 hash is available AND is up to date
 * for the shared file.
 *
 * NB: if the file is found to have changed, the background computation of
 * the SHA1 is requested.
 */
gboolean
sha1_hash_is_uptodate(shared_file_t *sf)
{
	filestat_t buf;

	shared_file_check(sf);

	if (!(sf->flags & SHARE_F_HAS_DIGEST))
		return FALSE;

	if (sf->flags & SHARE_F_RECOMPUTING)
		return FALSE;

	/*
	 * If there is a non-NULL `fi' entry, then this is a partially
	 * downloaded file that we are sharing.  Don't try to update its
	 * SHA1 by recomputing it!
	 *
	 * If it's a partial file, don't bother checking whether it exists.
	 * (if gone, we won't be able to serve it, that's all).  But partial
	 * files we serve MUST have known SHA1.
	 */

	if (shared_file_is_partial(sf)) {
		g_assert(sf->fi->sha1 != NULL);
		return TRUE;
	}

	if (-1 == stat(sf->file_path, &buf)) {
		g_warning("can't stat shared file #%d \"%s\": %s",
			sf->file_index, sf->file_path, g_strerror(errno));
		shared_file_set_sha1(sf, NULL);
		return FALSE;
	}

	if (too_big_for_gnutella(buf.st_size)) {
		g_warning("File is too big to be shared: \"%s\"", sf->file_path);
		shared_file_set_sha1(sf, NULL);
		return FALSE;
	}

	/*
	 * If file was modified since the last time we computed the SHA1,
	 * recompute it and tell them that the SHA1 we have might not be
	 * accurate.
	 */

	if (
			sf->mtime != buf.st_mtime ||
			sf->file_size + (fileoffset_t) 0 != buf.st_size + (filesize_t) 0
	) {
		g_warning("shared file #%d \"%s\" changed, recomputing SHA1",
			sf->file_index, sf->file_path);
		sf->flags |= SHARE_F_RECOMPUTING;
		sf->mtime = buf.st_mtime;
		sf->file_size = buf.st_size;
		request_sha1(sf);
		return FALSE;
	}

	return TRUE;
}

/**
 * Whether file is finished (i.e. either shared from the library or seeded).
 */
gboolean
shared_file_is_finished(const shared_file_t *sf)
{
	shared_file_check(sf);
	return NULL == sf->fi || 0 != (sf->fi->flags & FI_F_SEEDING);
}

gboolean
shared_file_is_partial(const shared_file_t *sf)
{
	shared_file_check(sf);
	return NULL != sf->fi;
}

filesize_t
shared_file_size(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->file_size;
}

guint32
shared_file_index(const shared_file_t *sf)
{
	shared_file_check(sf);

	g_assert(sf->file_index != 0);		/* Either PARTIAL_FILE or > 0 */

	return sf->file_index;
}

const struct sha1 *
shared_file_sha1(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->sha1;
}

const struct tth *
shared_file_tth(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->tth;
}

const char *
shared_file_name_nfc(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->name_nfc;
}

const char *
shared_file_name_canonic(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->name_canonic;
}

size_t
shared_file_name_nfc_len(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->name_nfc_len;
}

size_t
shared_file_name_canonic_len(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->name_canonic_len;
}

/**
 * Returns the relative path of the shared files unless there was none
 * or exposing relative paths is disabled.
 *
 * @return A string or NULL.
 */
const char *
shared_file_relative_path(const shared_file_t *sf)
{
	shared_file_check(sf);
	return GNET_PROPERTY(search_results_expose_relative_paths)
			? sf->relative_path
			: NULL;
}

/**
 * Get the pathname of a shared file.
 *
 * @param sf an initialized shared file.
 * @return	the full pathname of the shared file. The returned pointer is
 *			a string atom.
 */
const char *
shared_file_path(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->file_path;
}

time_t
shared_file_modification_time(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->mtime;
}

guint32
shared_file_flags(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->flags;
}

fileinfo_t *
shared_file_fileinfo(const shared_file_t *sf)
{
	shared_file_check(sf);
	return sf->fi;
}

const char *
shared_file_mime_type(const shared_file_t *sf)
{
	shared_file_check(sf);
	return mime_type_to_string(sf->mime_type);
}

void
shared_file_remove(shared_file_t *sf)
{
	shared_file_check(sf);

	shared_file_deindex(sf);
	if (0 == sf->refcnt) {
		shared_file_free(&sf);
	}
}

void
shared_file_set_path(shared_file_t *sf, const char *pathname)
{
	shared_file_check(sf);
	atom_str_change(&sf->file_path, pathname);
}

void
shared_file_from_fileinfo(fileinfo_t *fi)
{
	shared_file_t *sf;

	file_info_check(fi);
	g_assert(NULL == fi->sf);

	sf = shared_file_alloc();
	sf->flags = SHARE_F_HAS_DIGEST;
	sf->mtime = fi->last_flush;
	sf->sha1 = atom_sha1_get(fi->sha1);

	/* FIXME: DOWNLOAD_SIZE:
	 * Do we need to add anything here now that fileinfos can have an
	 *  unknown length? --- Emile
	 */

	sf->file_size = fi->size;
	sf->file_index = PARTIAL_FILE;	/* Until inserted in partial search table */

	/*
	 * Determine a proper human-readable name for the file.
	 * If it is an URN, look through the aliases.
	 */

	if (shared_file_set_names(sf, file_info_readable_filename(fi))) {
		shared_file_free(&sf);
		return;
	}

	sf->mime_type = mime_type_from_filename(sf->name_nfc);
	sf->file_path = atom_str_get(fi->pathname);

	sf->fi = fi;		/* Signals it's a partially downloaded file */
	fi->sf = shared_file_ref(sf);
}

/**
 * @returns the shared_file if we share a complete file bearing the given SHA1.
 * @returns NULL if we don't share a complete file, or SHARE_REBUILDING if the
 * set of shared file is being rebuilt.
 */
static shared_file_t *
shared_file_complete_by_sha1(const struct sha1 *sha1)
{
	shared_file_t *f;

	if (sha1_to_share == NULL)			/* Not even begun share_scan() yet */
		return SHARE_REBUILDING;

	f = g_tree_lookup(sha1_to_share, deconstify_gpointer(sha1));
	if (f) {
		shared_file_check(f);
	}

	if (!f || !sha1_hash_available(f)) {
		/*
		 * If we're rebuilding the library, we might not have parsed the
		 * file yet, so it's possible we have this URN but we don't know
		 * it yet.	--RAM, 12/10/2002.
		 */

		return share_rebuilding ? SHARE_REBUILDING : NULL;
	}

	return f;
}

/**
 * Take a given binary SHA1 digest, and return the corresponding
 * shared_file if we have it.
 *
 * @attention
 * NB: if the returned "shared_file" structure holds a non-NULL `fi',
 * then it means it is a partially shared file.
 *
 * @returns NULL if we don't share a complete file, or SHARE_REBUILDING if the
 * set of shared file is being rebuilt.
 */
shared_file_t *
shared_file_by_sha1(const struct sha1 *sha1)
{
	shared_file_t *f;

	f = shared_file_complete_by_sha1(sha1);

	/*
	 * If we don't share this file, or if we're rebuilding, and provided
	 * PFSP-server is enabled, look whether we don't have a partially
	 * downloaded file with this SHA1.
	 */

	if (f == NULL || f == SHARE_REBUILDING) {
		if (GNET_PROPERTY(pfsp_server) || GNET_PROPERTY(pfsp_rare_server)) {
			shared_file_t *sf = file_info_shared_sha1(sha1);
			if (sf != NULL) {
				if (GNET_PROPERTY(pfsp_rare_server)) {
					if (download_sha1_is_rare(sha1)) {
						f = sf;
					}
				} else {
					f = sf;
				}
			}
		}
	}
	if (f && SHARE_REBUILDING != f) {
		shared_file_check(f);
	}
	return f;
}

/**
 * Fill the supplied shared_file vector, holding sfcount entries, with the
 * most recent shared files we have in the library matching the supplied
 * media_mask (if non-zero) and which are less than SHARE_RECENT_THRESH
 * seconds old.
 *
 * @attention
 * No reference counting is done on the returned pointers, which must therefore
 * be used right away.
 *
 * @return the amount of entries filled in the vector.
 */
size_t
share_fill_newest(shared_file_t **sfvec, size_t sfcount, unsigned media_mask)
{
	int i;
	size_t j;

	g_assert(sfvec != NULL);
	g_assert(size_is_positive(sfcount));

	if (NULL == file_table)
		return 0;

	g_assert(files_scanned != 0);

	for (i = files_scanned - 1, j = 0; i >= 0 && j < sfcount; i--) {
		shared_file_t *sf = file_table[i];

		if (sf != NULL) {
			shared_file_check(sf);

			/* file_table[] is sorted by increasing mtime */

			if (delta_time(tm_time(), sf->mtime) > SHARE_RECENT_THRESH)
				break;		/* Deeper files will be older */

			if (media_mask != 0 && !shared_file_has_media_type(sf, media_mask))
				continue;

			sfvec[j++] = sf;
		}
	}

	return j;
}

/**
 * Is shared file belonging to the media types indicated by mask?
 */
gboolean
shared_file_has_media_type(const shared_file_t *sf, unsigned mask)
{
	unsigned type;

	shared_file_check(sf);

	type = pointer_to_uint(
		g_hash_table_lookup(share_media_types, int_to_pointer(sf->mime_type)));

	return 0 != (type & mask);
}

/**
 * Convenience routine: compute media type mask for a file name, corresponding
 * to the bits in the media type filter that must be set to return this type
 * of file.
 *
 * @return media mask associated to filename, 0 meaning we don't recognize
 * this type of file.
 */
unsigned
share_filename_media_mask(const char *filename)
{
	enum mime_type mime;
	const void *v;

	mime = mime_type_from_filename(filename);
	v = g_hash_table_lookup(share_media_types, int_to_pointer(mime));

	return pointer_to_uint(v);
}

/**
 * Get accessor for ``kbytes_scanned''
 */
guint64
shared_kbytes_scanned(void)
{
	return bytes_scanned / 1024;
}

/**
 * Get accessor for ``files_scanned''
 */
guint64
shared_files_scanned(void)
{
	return files_scanned;
}

/**
 * Callout queue callback to initiate a QRP rebuild after a partial file
 * insertion or removal.
 */
static void
share_qrp_rebuild(cqueue_t *unused_cq, void *unused_data)
{
	(void) unused_cq;
	(void) unused_data;

	share_qrp_rebuild_ev = NULL;

	if (share_update_qrp()) {
		if (GNET_PROPERTY(share_debug) > 1) {
			g_debug("SHARE launched background QRP recomputation (%s)",
				share_can_answer_partials() ?
					"with partial files" : "library only");
		}
	} else {
		if (GNET_PROPERTY(share_debug) > 1) {
			g_debug("SHARE deferring background QRP recomputation (%s)",
				share_can_answer_partials() ?
					"with partial files" : "library only");
		}
		share_qrp_rebuild_ev = cq_main_insert(1000, share_qrp_rebuild, NULL);
	}
}

/**
 * Request asynchronous partial file table (for pattern matching) and QRP
 * table rebuild if necessary.
 */
static void
share_qrp_rebuild_if_needed(void)
{
	if (NULL == share_qrp_rebuild_ev && share_can_answer_partials())
		share_qrp_rebuild_ev = cq_main_insert(1000, share_qrp_rebuild, NULL);
}

/**
 * Records partial file entry.
 */
void
share_add_partial(const shared_file_t *sf)
{
	g_assert(shared_file_is_partial(sf));

	if (hash_list_contains(partial_files, sf))
		return;

	hash_list_append(partial_files, sf);

	/*
	 * We added a new partial file, we need to rebuild the QRP table.
	 * Do that asynchronously in case we're called frequently from a loop,
	 * for instance at startup or when many new files are downloaded.
	 */

	share_qrp_rebuild_if_needed();

	if (GNET_PROPERTY(share_debug) > 1)
		g_debug("SHARE added partial file \"%s\"", shared_file_path(sf));
}

/**
 * Removes partial file entry.
 */
void
share_remove_partial(const shared_file_t *sf)
{
	g_assert(shared_file_is_partial(sf));

	if (NULL == hash_list_remove(partial_files, sf))
		return;

	/*
	 * We removed a partial file, we need to rebuild the QRP table.
	 */

	share_qrp_rebuild_if_needed();

	if (GNET_PROPERTY(share_debug) > 1)
		g_debug("SHARE removed partial file \"%s\"", shared_file_path(sf));
}

/**
 * Whenever support for partial file sharing or partial result answering
 * changes, rebuild the QRP and matching tables, asynchronously.
 */
void
share_update_matching_information(void)
{
	if (NULL == share_qrp_rebuild_ev)
		share_qrp_rebuild_ev = cq_main_insert(1, share_qrp_rebuild, NULL);
}

/**
 * Initialization of the sharing library.
 */
G_GNUC_COLD void
share_init(void)
{
	size_t i;

	huge_init();
	qrp_init();
	qhit_init();
	oob_init();
	oob_proxy_init();
	share_special_init();

	/*
	 * We allocate an empty search_table, which will be de-allocated when we
	 * call share_scan().  Why do we do this?  Because it ensures the table
	 * is correctly setup empty, until we do call share_scan() for the first
	 * time (the call is delayed until the GUI is up).
	 *
	 * Since we will start processing network packets, we will have a race
	 * condition window if we get a Query message before having started
	 * the share_scan().  Creating the table right now prevents adding an
	 * extra test at the top of st_search().
	 *		--RAM, 15/08/2002.
	 */

	search_table = st_create();

	/*
	 * Intialize partial file querying structures (so that queries can
	 * be applied to partial files).
	 */

	partial_files = hash_list_new(pointer_hash_func, NULL);
	partial_table = st_create();

	/*
	 * Create the hash table yielding the media type flags from a MIME type.
	 */

	share_media_types = g_hash_table_new(NULL, NULL);

	for (i = 0; i < G_N_ELEMENTS(media_type_map); i++) {
		g_hash_table_insert(share_media_types,
			int_to_pointer(media_type_map[i].type),
			int_to_pointer(media_type_map[i].flags));
	}
}

/* vi: set ts=4 sw=4 cindent: */
