From f3c33396cca2e104221435f17d6f6c3337395466 Mon Sep 17 00:00:00 2001
From: lars <lars@systemausfall.org>
Date: Tue, 6 Jan 2009 10:46:00 +0000
Subject: [PATCH] OSM scripts: * moved the bulk mailing script to a different
 location * added a script to improve the output of the OSM-QA data for MV

---
 osm/bulk-message/generate_send_message_url.py | 71 +++++++++++++++++
 osm/bulk-message/html_footer.inc              |  4 +
 osm/bulk-message/html_header.inc              |  3 +
 osm/bulk-message/info.txt                     | 36 +++++++++
 osm/bulk-message/username_to_number.sh        | 20 +++++
 osm/qa/generate_unmapped_places.sh            | 77 +++++++++++++++++++
 6 files changed, 211 insertions(+)
 create mode 100755 osm/bulk-message/generate_send_message_url.py
 create mode 100644 osm/bulk-message/html_footer.inc
 create mode 100644 osm/bulk-message/html_header.inc
 create mode 100644 osm/bulk-message/info.txt
 create mode 100755 osm/bulk-message/username_to_number.sh
 create mode 100755 osm/qa/generate_unmapped_places.sh

diff --git a/osm/bulk-message/generate_send_message_url.py b/osm/bulk-message/generate_send_message_url.py
new file mode 100755
index 0000000..47a5712
--- /dev/null
+++ b/osm/bulk-message/generate_send_message_url.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+import sys
+
+SEND_MESSAGE_URL = "http://openstreetmap.org/message/new/%d"
+
+# BEWARE: all non-7-bit ascii characters (umlauts and so on ...) are ignored
+message_title = "Neue regionale Mailingliste fuer Mecklenburg-Vorpommern"
+message_body_template = """Hallo %s,
+
+du erhaeltst diese Nachricht, da du aus Mecklenburg-Vorpommern kommst oder weil du mehr als 500 Nodes bzw. Wege in MV bearbeitet hast.
+
+Seit heute gibt es eine neue regionale OSM-Mailingliste fuer Mecklenburg-Vorpommern.
+
+Die Liste wird hoffentlich ein hilfreiches Medium zur Diskussion lokaler Themen, zur Planung von Mapping-Parties oder einfach nur zur gegenseitigen Wahrnehmung der verstreuten Nachbarn werden.
+
+Wenn du am Fortschritt von OSM in Mecklenburg-Vorpommern interessiert bist, dann trage dich doch bitte in die neue Mailingliste ein:
+http://lists.openstreetmap.de/cgi-bin/mailman/listinfo/meckpomm
+
+Vielleicht schaffen wir es ja trotz der derzeit unwirtlichen Wetterzustaende in Kuerze mal eine Mapping-Party zu organisieren, um langsam die zahlreichen weissen Flecken von MV zu verschoenern ...
+
+Eventuell bis bald auf der Mailingliste,
+Lars
+
+PS: falls dich die regionale Mailingliste nicht interessiert, dann ignoriere diese Nachricht einfach. Du wirst keine weiteren Nachrichten mehr erhalten.
+"""
+
+# return a html encoded representation (e.g. %20 for a space)
+# ignore 8-bit ascii characters
+def mask_text(text):
+	result = ""
+	for char in text:
+		num = ord(char)
+		if num > 127:
+			# non-ascii (7-bit)
+			pass
+		elif num > 15:
+			# two digits
+			result += "%" + hex(num)[2:]
+		else:
+			# one digit
+			result += "%0" + hex(num)[2:]
+	return result
+
+
+def generate_URL(username, user_id):
+	message = mask_text(message_body_template % username)
+	title = mask_text(message_title)
+	return (SEND_MESSAGE_URL + "?message[title]=%s&message[body]=%s") % \
+			(user_id, title, message)
+
+
+def generate_html_link(username, link):
+	return "<li><a href=%s>%s</a></li>\n" % (link, username)
+
+
+if __name__ == "__main__":
+	if len(sys.argv) == 3:
+		username = sys.argv[1]
+		try:
+			user_id = int(sys.argv[2])
+		except ValueError:
+			sys.stderr.write("invalid user id: %s\n" % sys.argv[2])
+			sys.exit(2)
+		print generate_html_link(username, generate_URL(username, user_id))
+	else:
+		sys.stderr.write("invalid input\n")
+		sys.exit(1)
+
+	sys.exit(0)
+
diff --git a/osm/bulk-message/html_footer.inc b/osm/bulk-message/html_footer.inc
new file mode 100644
index 0000000..8fdf4c6
--- /dev/null
+++ b/osm/bulk-message/html_footer.inc
@@ -0,0 +1,4 @@
+</ul>
+</body>
+</html>
+
diff --git a/osm/bulk-message/html_header.inc b/osm/bulk-message/html_header.inc
new file mode 100644
index 0000000..844d8f7
--- /dev/null
+++ b/osm/bulk-message/html_header.inc
@@ -0,0 +1,3 @@
+<html>
+<body>
+<ul>
diff --git a/osm/bulk-message/info.txt b/osm/bulk-message/info.txt
new file mode 100644
index 0000000..5c27b7b
--- /dev/null
+++ b/osm/bulk-message/info.txt
@@ -0,0 +1,36 @@
+How to send mails to many OSM users?
+====================================
+
+1) generate a list of OSM usernames, that should be the target of a mail
+
+2) retrieve the user id for each username (via the shell script)
+
+3) edit the mail text in the python script
+
+4) run the following command:
+	( \
+		cat html_header.inc; cat FILE_WITH_USERS_AND_IDS | while read line; \
+		do username="$(echo "$line" | cut -f 1)"; \
+			user_id="$(echo "$line" | cut -f 2)"; \
+			./generate_send_message_url.py "$username" "$user_id"; \
+			done; \
+		cat html_footer.inc \
+	) >mail_links.html
+
+5) open the html file in your favourite browser
+
+6) login to your openstreetmap account
+
+7) click on each of the links in the local html file to send a single message
+
+
+
+How to get all active mappers of an area?
+=========================================
+
+1) download a snapshot of the region from http://geofabrik.de
+
+2) extract the users and sort them by the number of objects:
+	grep user= mecklenburg-vorpommern.osm | cut -d '"' -f 6 \
+		| sort | uniq -c | sort -nr
+
diff --git a/osm/bulk-message/username_to_number.sh b/osm/bulk-message/username_to_number.sh
new file mode 100755
index 0000000..d862c54
--- /dev/null
+++ b/osm/bulk-message/username_to_number.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# retrieve the OSM-userid for a given username
+#
+# the script reads single usernames from stdin and outputs the username and
+# the corresponding ID
+#
+
+
+USER_URL="http://openstreetmap.org/user"
+
+get_user_id() {
+	wget -q -O - "$USER_URL/$1" | grep "/message/new/" | cut -d '"' -f 2 | cut -d / -f 4
+}
+
+while read username
+  do	echo -ne "$username\t"
+		get_user_id "$username"
+ done
+
diff --git a/osm/qa/generate_unmapped_places.sh b/osm/qa/generate_unmapped_places.sh
new file mode 100755
index 0000000..a76bd2a
--- /dev/null
+++ b/osm/qa/generate_unmapped_places.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+
+# original data
+QUALITY_URL="http://www.gary68.de/osm/qa/unmapped/mecklenburg-vorpommern.htm"
+# where to get the number of inhabitants
+WIKIPEDIA_URL="http://de.wikipedia.org/wiki"
+PLACE_TYPE="(town|city)"
+# some wikipedia pages have different names
+# remove ALL non-letters (7-bit) characters from the _first_ column
+PLACE_NAME_MAPPING="Feldberg	Feldberger_Seenlandschaft
+BoizenburgElbe	Boizenburg
+FrstenbergHavel	F%C3%BCrstenberg/Havel
+Goldberg	Goldberg_(Mecklenburg)
+Malchow	Malchow_(Mecklenburg)
+OstseebadRerik	Rerik
+RbelMritz	Röbel
+Strasburg	Strasburg_(Uckermark)
+Tessin	Tessin_(bei_Rostock)
+Wesenberg	Wesenberg_(Mecklenburg)
+Zarrentin	Amt_Zarrentin"
+
+
+get_place_data() {
+	echo '<html><body><table border="1">'
+	wget --quiet --output-document - "$QUALITY_URL" \
+			| sed -n "1,/Details all information/p" \
+			| grep -E -A 4 -B 2 "^<td>$PLACE_TYPE</td>$" \
+			| grep -v "^--$"
+	echo '</table></body></html>'
+}
+
+
+get_place_size() {
+	wget --quiet --output-document - "$WIKIPEDIA_URL/$1" \
+			| grep -A 1 "^<td>Einwohner:</td>$" \
+			| tail -1 | cut -d ">" -f 2 | cut -d "<" -f 1 \
+			| sed s/[^0-9]//g
+}
+
+
+normalize_place_name() {
+	local simple_name="$(echo "$1" | sed s/[^a-zA-Z]//g)"
+	local new_name="$(echo "$PLACE_NAME_MAPPING" | grep "^$simple_name" | cut -f 2)"
+	if test -z "$new_name"
+	  then	echo "$1"
+	  else	echo "$new_name"
+	 fi
+}
+
+
+get_place_inhabitants() {
+	local previous_line
+	local place_name
+	local place_size
+	local place_nodes
+	while read line
+	  do	if echo "$line" | grep -q -E "<td>$PLACE_TYPE</td>"
+			  then	place_name="$(echo "$previous_line" | cut -d ">" -f 2 | cut -d "<" -f 1)"
+					place_name="$(normalize_place_name "$place_name")"
+					place_size="$(get_place_size "$place_name")"
+			  else	if echo "$line" | grep -q "^</tr>$"
+					  then	echo "<td>$place_size</td>"
+							place_nodes="$(echo "$previous_line" | cut -d ">" -f 2 | cut -d "<" -f 1)"
+							echo "<td>$(echo "scale=2;$place_size/$place_nodes" | bc)</td>"
+					 fi
+			 fi
+			echo "$line"
+			previous_line="$line"
+	 done
+}
+
+
+# we need the default locales - otherwise 'sed' will not filter umlauts
+export LANG=
+
+get_place_data | get_place_inhabitants
+