瀹氫箟錛?
鐏扮緤緹?nbsp; 錛堟棤涓昏鐨勭敤鎴風兢浣擄級
榛戠緤 錛?瀵硅嚜宸遍渶瑕佷粈涔堟湁鏄庣‘鐨勮璇嗭紝鎴戜滑涓鑸О涓轟笓瀹剁敤鎴楓?nbsp; 錛?br />
1. 鍖哄垎 鐏幫紙鏃犱富瑙侊級 榛?緹婄兢
2.
user session 鍏寵仈 #褰?鍏寵仈鍏崇郴緇存姢浣跨敤 鐢ㄦ埛鐨勪細璇滻D錛堢敤鎴蜂笉鍚屽績鎯咃紝璧峰鍦ㄦ暟鎹腑灝卞簲璇ユ槸涓嶅悓鍒嗙被鐨勶級
user 鎺ㄨ崘 # 鑰屾帹鑽愬嚭浜у搧 榪樻槸 璺?鐢ㄦ埛鍞竴緙栧彿鏈夊叧
#鍦ㄦ帹鑽愪腑闇瑕佹弿榪?nbsp; 鐢ㄦ埛鐨勫瑙掑害 闂
3.
铔姏鎺ㄨ崘錛?鍏ㄦ暟鎹?錛涙弿榪板垵鏈熸竻媧楀悗鐨勬暟鎹?錛?閫傚悎 浜у搧鍏寵仈
娓呮礂鍚庢湡鐨勬暟鎹紙鍖呭惈鐢ㄦ埛澶氱淮搴︽弿榪幫級 閫傚悎 鐢ㄦ埛鍏寵仈
4.
涓撳璺熼殢鎺ㄨ崘
鎻忚堪錛?br /> 鐢ㄦ埛鍒嗙被 鎵懼埌榛戠壞緹?
鎵懼埌 涓緹ょ伆緇電緤 鍜?涓鍙粦緇電緤鐨勫叧鑱斿叧緋?br /> 璁?涓緹ょ伆緇電緤 鍙互鐪?榛戠壞緹?鍔ㄤ綔
]]>
姣斿錛氭垜甯屾湜鍘熷鏃ュ織鏁版嵁榪涜鏌愪竴瀛楁鐨勬帓搴?鏄笉鏄緢綆鍗?銆?
鏈変漢璇?nbsp; - 鏁版嵁瀵煎叆鏁版嵁搴?load into table ... 錛?select order by 銆備箣
榪樻湁浜鴻 - linux sort -n...
鎭╋紒寰堝ソ錛屼笅闈㈡垜浠澶у皬涓?1TB 鐨勬暟鎹紑濮嬭繘琛岃繖涓畝鍗曠殑鎿嶄綔 -- 鍌葷溂浜?錛侊紒
鍏充簬鎸栨帢 - TB 綰у埆鐨勬暟閲忓湪鎴戠洰鍓嶅涔犳寲鎺樹笉鍒板崐騫達紝灝遍亣鍒拌繃3-4嬈′箣澶?br />
瑙ｅ喅鍔炴硶:
瀵逛簬榪欎釜闂 - 鎴戠幇鍦ㄥ笇鏈涜兘鏈変釜 澶х殑閾捐〃 - 錛堝ぇ鍒板唴瀛樿涓嶄笅錛?/strong>錛?br /> 閾捐〃涓殑struct 緇撴瀯涓?:
>> 鎺掑簭灞炴ф枃浠跺綊灞?br /> >> 鎺掑簭灞炴ф暣鏉℃暟鎹湪鏂囦歡涓殑 璧峰浣嶇疆 - 緇撴潫浣嶇疆
>> 鍦ㄦ帓搴忎腑鐨勬帓浣?錛?閾捐〃緇撴瀯,鍙鍏ユ瘮鑷繁灝忕殑 灞炴у湪姝ら摼琛ㄧ殑浣嶇疆 錛?br />
姣斿 :
1. 鏂囦歡1鍐呭 =>
璇存槑:
瀹屾暣鏁版嵁鎻忚堪 : 姝ゆ暟鎹湪鏂囦歡涓殑 璧峰浣嶇疆錛堝綋鐒舵槸閫氳繃紼嬪簭鍙栧緱鐨勶紝榪欎負浜嗘柟渚挎垜鏍囧嚭錛?/strong>
..c
. 0 - 22
..a. 23 - 55
..b. 56- 76
..d. 77 - 130
..f. 131 - 220
..e. 221 - 243
瀹屾暣鏁版嵁鎻忚堪 : 姝ゆ暟鎹湪鏂囦歡涓殑 璧峰浣嶇疆錛堝綋鐒舵槸閫氳繃紼嬪簭鍙栧緱鐨勶紝榪欎負浜嗘柟渚挎垜鏍囧嚭錛?/strong>
..c
. 0 - 22..a
. 23 - 55..b
. 56- 76..d
. 77 - 130..f
. 131 - 220..e
. 221 - 2432. 鏁版嵁緇撴瀯棰勫紑絀洪棿 100 byte
3. 鏂囦歡瀛樺偍鍦ㄦ弿榪?: # 閾捐〃鎺掑簭鎴戝氨涓嶄粙緇嶄簡錛屾暟鎹粨鏋勭殑鏈鍩烘湰鎶鑳斤紝淇敼鏁版嵁緇撴瀯涓殑姣旇嚜宸卞皬鐨勬寚鍚?
鎴戣繖灝辯粰鍑虹粨鏋?br /> { /tmp/鏂囦歡1, 0-22 , 300 } #璇存槑 c 錛?鍦ㄩ摼琛ㄤ綅緗?0
{ /tmp/鏂囦歡1, 23-55 , 200 } # a 錛?100
{ /tmp/鏂囦歡1, 56-76 , 0 } # b : 200
{ /tmp/鏂囦歡1, 77-130 , 500 } # d : 300
{ /tmp/鏂囦歡1, 131-220 , } # f : 400
{ /tmp/鏂囦歡1, 221-243 , 400 } # e : 500
4. 鍊掑彊杈撳嚭 鐢卞皬鍒板埌
鍋囪棰勫瓨鏈灝?涓?nbsp; 200 閾捐〃浣嶇疆
鎵懼嚭 浣跨敤 open /tmp/鏂囦歡1
騫朵嬌鐢?seek 鏂囦歡娓告爣 瀹氫綅 23-55 鍙栧嚭 ..a...
鏍規嵁 閾捐〃涓?200 鍒?seek 56 76 鍙栧嚭 ..b...
絳夌瓑
褰撶劧 涓婇潰
鏁版嵁緇撴瀯浣犲彲浠ヤ嬌鐢?鍙屽悜閾捐〃錛?btree , 綰㈤粦 , 鏂愭嘗閭ｅ銆傘傘傦紙 鏁版嵁緇撴瀯緇堜簬鎰熻鏈夌敤浜嗭紝涓嶆瀴璐規垜鑰冪殑杞瘉鍟婏紒錛?br />
閫氳繃璇存槑錛屾垜榪?緇欏ぇ瀹舵彁渚涗釜 鍙兘闇瑕佺殑 鎶鏈粏鑺?(py),涓嶈凍涔嬪 嬈㈣繋鎷嶇爾錛侊紒
1. 浜岃繘鍒舵枃浠?緇撴瀯鍖?鍐欙紝淇敼
#鎸囧畾淇敼 190 byte 澶勭殑 鍐呭
import os
from struct import *
fd = os.open( "pack1.txt", os.O_RDWR|os.O_CREAT )
ss = pack('ii11s', 3, 4, 'google')
os.lseek(fs, len(ss)*10, 0)
os.write(fs,ss)
os.fsync(fs)
#os.close( fs )
import os
from struct import *
fd = os.open( "pack1.txt", os.O_RDWR|os.O_CREAT )
ss = pack('ii11s', 3, 4, 'google')
os.lseek(fs, len(ss)*10, 0)
os.write(fs,ss)
os.fsync(fs)
#os.close( fs )
2. seek 鎸囧畾浣嶇疆緇撴瀯鍖栬鍙?br />
from struct import *
file_object = open('pack1.txt', 'rb')
def ts(si,ss=len(ss)):
file_object.seek(si*ss)
chunk = file_object.read(ss)
a,b,c=unpack('ii11s', chunk )
print a,b,c
ts(10)
#杈撳嚭 3 4 google
1. 鍏朵粬璇█鐨?浣跨敤
struct 緇撴瀯瀹氫箟 ,鍦?python 涓?浣跨敤 struct 鍖咃紝榪欐牱搴忓垪鍑烘潵鐨勬暟鎹埌鏂囦歡涓叾浠栬璦涔熷彲浠ヤ嬌鐢?
鍙傝? http://www.pythonid.com/bbs/archiver/?tid-285.html
pack1.py
from struct import *
# i 涓?int錛?錛?nbsp; 11s 涓洪鐣?11 浣嶇疆 鐨?string
# 姝ゆ暟鎹被鍨?涓?19 byte ss = pack('ii11s', 1, 2, 'hello world')
f = open("pack1.txt", "wb")
f.write(ss)
f.close()
涓婇潰鐨勪唬鐮佸線C鐨勭粨鏋勪腑鍐欏叆鏁版嵁錛岀粨鏋勫寘鎷袱涓暣鍨嬪拰涓涓瓧絎︿覆銆?br /> pack1.c
#include <stdio.h>
#include <string.h>
struct AA
{
int a;
int b;
char c[64];
};
int main()
{
struct AA aa;
FILE *fp;
int size, readsize;
memset(&aa, 0, sizeof(struct AA));
fp = fopen("pack1.txt", "rb");
if (NULL == fp) {
printf("open file error!"n");
return 0;
}
readsize = sizeof(struct AA);
printf("readsize: %d"n", readsize);
size = fread(&aa, 1, readsize, fp);
printf("read: %d"n", size);
printf("a=%d, b=%d, c=%s"n", aa.a, aa.b, aa.c);
fclose(fp);
return 0;
}
緇撴灉杈撳嚭:
C:"Documents and Settings"lky"妗岄潰"dataStructure>a
readsize: 72
read: 57
a=1, b=2, c=hello word
from struct import *
# i 涓?int錛?錛?nbsp; 11s 涓洪鐣?11 浣嶇疆 鐨?string
# 姝ゆ暟鎹被鍨?涓?19 byte ss = pack('ii11s', 1, 2, 'hello world')
f = open("pack1.txt", "wb")
f.write(ss)
f.close()
涓婇潰鐨勪唬鐮佸線C鐨勭粨鏋勪腑鍐欏叆鏁版嵁錛岀粨鏋勫寘鎷袱涓暣鍨嬪拰涓涓瓧絎︿覆銆?br /> pack1.c
#include <stdio.h>
#include <string.h>
struct AA
{
int a;
int b;
char c[64];
};
int main()
{
struct AA aa;
FILE *fp;
int size, readsize;
memset(&aa, 0, sizeof(struct AA));
fp = fopen("pack1.txt", "rb");
if (NULL == fp) {
printf("open file error!"n");
return 0;
}
readsize = sizeof(struct AA);
printf("readsize: %d"n", readsize);
size = fread(&aa, 1, readsize, fp);
printf("read: %d"n", size);
printf("a=%d, b=%d, c=%s"n", aa.a, aa.b, aa.c);
fclose(fp);
return 0;
}
緇撴灉杈撳嚭:
C:"Documents and Settings"lky"妗岄潰"dataStructure>a
readsize: 72
read: 57
a=1, b=2, c=hello word
鏈鍚庣綏鍡︿笅錛?/strong>
鑳界敤鏁版嵁緇撴瀯浜嗭紝寰堝涓滆タ閮藉彲浠ユ牴鎹嚜宸遍昏緫瀹氬埗 瀛樺偍寰堟柟渚?銆?涓嶅啀鍙?鍏崇郴鏁版嵁搴?, key 鏁版嵁搴?鎴?mapreduce 鐨勯檺鍒?
鍙傝?
http://docs.python.org/library/struct.html#module-struct #瀹樻柟struct 鍖?璇存槑
http://blog.csdn.net/JGood/archive/2009/06/22/4290158.aspx # 浣跨敤 struct 鐨勫墠杈堢暀涓嬬殑
http://www.tutorialspoint.com/python/os_lseek.htm #涓涓皬demo
Python澶╁ぉ緹庡懗(17) - open璇誨啓鏂囦歡
]]>
鏁版嵁鎸栨帢鎵鍙戠幇鐨勭煡璇嗘渶甯歌鐨勬湁浠ヤ笅鍥涚被錛?
騫? 涔夌煡璇嗙殑鍙戠幇鏂規硶鍜屽疄鐜版妧鏈湁寰堝錛屽鏁版嵁绔嬫柟浣撱侀潰鍚戝睘鎬х殑褰掔害絳夈傛暟鎹珛鏂逛綋榪樻湁鍏朵粬涓浜涘埆鍚嶏紝濡?#8220;澶氱淮鏁版嵁搴?#8221;銆?#8220;瀹炵幇瑙嗗浘”銆?#8220;OLAP"絳夈傝 鏂規硶鐨勫熀鏈濇兂鏄疄鐜版煇浜涘父鐢ㄧ殑浠ｄ環杈冮珮鐨勮仛闆嗗嚱鏁扮殑璁＄畻錛岃濡傝鏁般佹眰鍜屻佸鉤鍧囥佹渶澶у肩瓑錛屽茍灝嗚繖浜涘疄鐜拌鍥懼偍瀛樺湪澶氱淮鏁版嵁搴撲腑銆傛棦鐒跺緢澶氳仛闆嗗嚱鏁伴渶緇? 甯擱噸澶嶈綆楋紝閭ｄ箞鍦ㄥ緇存暟鎹珛鏂逛綋涓瓨鏀鵑鍏堣綆楀ソ鐨勭粨鏋滃皢鑳戒繚璇佸揩閫熷搷搴旓紝騫跺彲鐏墊椿鍦版彁渚涗笉鍚岃搴﹀拰涓嶅悓鎶借薄灞傛涓婄殑鏁版嵁瑙嗗浘銆傚彟涓縐嶅箍涔夌煡璇嗗彂鐜版柟娉? 鏄姞鎷垮ぇSimonFraser澶у鎻愬嚭鐨勯潰鍚戝睘鎬х殑褰掔害鏂規硶銆傝繖縐嶆柟娉曚互綾籗QL璇█琛ㄧず鏁版嵁鎸栨帢鏌ヨ錛屾敹闆嗘暟鎹簱涓殑鐩稿叧鏁版嵁闆嗭紝鐒跺悗鍦ㄧ浉鍏蟲暟鎹泦涓? 搴旂敤涓緋誨垪鏁版嵁鎺ㄥ箍鎶鏈繘琛屾暟鎹帹騫匡紝鍖呮嫭灞炴у垹闄ゃ佹蹇墊爲鎻愬崌銆佸睘鎬ч槇鍊兼帶鍒躲佽鏁板強鍏朵粬鑱氶泦鍑芥暟浼犳挱絳夈?/td>
鏁版嵁鍒嗙被榪樻湁緇熻銆佺矖緋欓泦錛圧oughSet錛夌瓑鏂規硶銆傜嚎鎬у洖褰掑拰綰挎ц鯨鍒垎鏋愭槸鍏稿瀷鐨勭粺璁℃ā鍨嬨備負闄嶄綆鍐崇瓥鏍戠敓鎴愪唬浠鳳紝浜轟滑榪樻彁鍑轟簡涓縐嶅尯闂村垎綾誨櫒銆傛渶榪戜篃鏈変漢鐮旂┒浣跨敤紲炵粡緗戠粶鏂規硶鍦ㄦ暟鎹簱涓繘琛屽垎綾誨拰瑙勫垯鎻愬彇銆?/td>
鐩? 鍓嶏紝鏃墮棿搴忓垪棰勬祴鏂規硶鏈夌粡鍏哥殑緇熻鏂規硶銆佺緇忕綉緇滃拰鏈哄櫒瀛︿範絳夈?968騫碆ox鍜孞enkins鎻愬嚭浜嗕竴濂楁瘮杈冨畬鍠勭殑鏃墮棿搴忓垪寤烘ā鐞嗚鍜屽垎鏋愭柟娉曪紝榪欎簺 緇忓吀鐨勬暟瀛︽柟娉曢氳繃寤虹珛闅忔満妯″瀷錛屽鑷洖褰掓ā鍨嬨佽嚜鍥炲綊婊戝姩騫沖潎妯″瀷銆佹眰鍜岃嚜鍥炲綊婊戝姩騫沖潎妯″瀷鍜屽鑺傝皟鏁存ā鍨嬬瓑錛岃繘琛屾椂闂村簭鍒楃殑棰勬祴銆傜敱浜庡ぇ閲忕殑鏃墮棿搴忓垪 鏄潪騫崇ǔ鐨勶紝鍏剁壒寰佸弬鏁板拰鏁版嵁鍒嗗竷闅忕潃鏃墮棿鐨勬帹縐昏屽彂鐢熷彉鍖栥傚洜姝わ紝浠呬粎閫氳繃瀵規煇孌靛巻鍙叉暟鎹殑璁粌錛屽緩绔嬪崟涓鐨勭緇忕綉緇滈嫻嬫ā鍨嬶紝榪樻棤娉曞畬鎴愬噯紜殑棰勬祴浠? 鍔°備負姝わ紝浜轟滑鎻愬嚭浜嗗熀浜庣粺璁″鍜屽熀浜庣簿紜х殑鍐嶈緇冩柟娉曪紝褰撳彂鐜扮幇瀛橀嫻嬫ā鍨嬩笉鍐嶉傜敤浜庡綋鍓嶆暟鎹椂錛屽妯″瀷閲嶆柊璁粌錛岃幏寰楁柊鐨勬潈閲嶅弬鏁幫紝寤虹珛鏂扮殑妯″瀷銆? 涔熸湁璁稿緋葷粺鍊熷姪騫惰綆楁硶鐨勮綆椾紭鍔胯繘琛屾椂闂村簭鍒楅嫻嬨?
閲囩敤涓婅堪鎶鏈殑鏌愪簺涓撻棬鐨勫垎鏋愬伐鍏峰凡緇忓彂灞曚簡澶х害鍗佸勾鐨勫巻鍙詫紝涓嶈繃榪欎簺宸ュ叿鎵闈㈠鐨勬暟鎹噺閫氬父杈冨皬銆傝岀幇鍦ㄨ繖浜涙妧鏈凡緇忚鐩存帴闆嗘垚鍒拌澶氬ぇ鍨嬬殑宸ヤ笟鏍囧噯鐨勬暟鎹粨搴撳拰鑱旀満鍒嗘瀽緋葷粺涓幓浜嗐?
]]>
娉ㄦ剰
鐩墠 streaming 瀵?linux pipe #涔熷氨鏄?cat |wc -l 榪欐牱鐨勭閬?涓嶆敮鎸侊紝浣嗕笉濡ㄧ鎴戜滑浣跨敤perl,python 琛屽紡鍛戒護錛侊紒
鍘熻瘽鏄?錛?br /> Can I use UNIX pipes? For example, will -mapper "cut -f1 | sed s/foo/bar/g" work?
Currently this does not work and gives an "java.io.IOException: Broken pipe" error.
This is probably a bug that needs to be investigated.
浣嗗鏋滀綘鏄己鐑堢殑 linux shell pipe 鍙戠儳鍙?錛?鍙傝冧笅闈?br /> $> perl -e 'open( my $fh, "grep -v null tt |sed -n 1,5p |");while ( <$fh> ) {print;} '
#涓嶈繃鎴戞病嫻嬭瘯閫氳繃 錛侊紒
鐜 錛歨adoop-0.18.3
$> find . -type f -name "*streaming*.jar"
./contrib/streaming/hadoop-0.18.3-streaming.jar
嫻嬭瘯鏁版嵁錛?br />
-bash-3.00$ head tt
null false 3702 208100
6005100 false 70 13220
6005127 false 24 4640
6005160 false 25 4820
6005161 false 20 3620
6005164 false 14 1280
6005165 false 37 7080
6005168 false 104 20140
6005169 false 35 6680
6005240 false 169 32140
......
null false 3702 208100
6005100 false 70 13220
6005127 false 24 4640
6005160 false 25 4820
6005161 false 20 3620
6005164 false 14 1280
6005165 false 37 7080
6005168 false 104 20140
6005169 false 35 6680
6005240 false 169 32140
......
榪愯錛?br />
c1=" perl -ne 'if(/.*\t(.*)/){\$sum+=\$1;}END{print \"\$sum\";}' "
# 娉ㄦ剰 榪欓噷 $ 瑕佸啓鎴?\$ " 鍐欐垚 \"
echo $c1; # 鎵撳嵃杈撳嚭 perl -ne 'if(/.*"t(.*)/){$sum+=$1;}END{print $sum;}'
hadoop jar hadoop-0.18.3-streaming.jar
-input file:///data/hadoop/lky/jar/tt
-mapper "/bin/cat"
-reducer "$c1"
-output file:///tmp/lky/streamingx8
# 娉ㄦ剰 榪欓噷 $ 瑕佸啓鎴?\$ " 鍐欐垚 \"
echo $c1; # 鎵撳嵃杈撳嚭 perl -ne 'if(/.*"t(.*)/){$sum+=$1;}END{print $sum;}'
hadoop jar hadoop-0.18.3-streaming.jar
-input file:///data/hadoop/lky/jar/tt
-mapper "/bin/cat"
-reducer "$c1"
-output file:///tmp/lky/streamingx8
緇撴灉:
cat /tmp/lky/streamingx8/*
1166480
鏈湴榪愯杈撳嚭:
perl -ne 'if(/.*"t(.*)/){$sum+=$1;}END{print $sum;}' < tt
1166480
緇撴灉姝ｇ‘!!!!
鍛戒護鑷甫鏂囨。錛?br />
-bash-3.00$ hadoop jar hadoop-0.18.3-streaming.jar -info
09/09/25 14:50:12 ERROR streaming.StreamJob: Missing required option -input
Usage: $HADOOP_HOME/bin/hadoop [--config dir] jar \
$HADOOP_HOME/hadoop-streaming.jar [options]
Options:
-input <path> DFS input file(s) for the Map step
-output <path> DFS output directory for the Reduce step
-mapper <cmd|JavaClassName> The streaming command to run
-combiner <JavaClassName> Combiner has to be a Java class
-reducer <cmd|JavaClassName> The streaming command to run
-file <file> File/dir to be shipped in the Job jar file
-dfs <h:p>|local Optional. Override DFS configuration
-jt <h:p>|local Optional. Override JobTracker configuration
-additionalconfspec specfile Optional.
-inputformat TextInputFormat(default)|SequenceFileAsTextInputFormat|JavaClassName Optional.
-outputformat TextOutputFormat(default)|JavaClassName Optional.
-partitioner JavaClassName Optional.
-numReduceTasks <num> Optional.
-inputreader <spec> Optional.
-jobconf <n>=<v> Optional. Add or override a JobConf property
-cmdenv <n>=<v> Optional. Pass env.var to streaming commands
-mapdebug <path> Optional. To run this script when a map task fails
-reducedebug <path> Optional. To run this script when a reduce task fails
-cacheFile fileNameURI
-cacheArchive fileNameURI
-verbose
09/09/25 14:50:12 ERROR streaming.StreamJob: Missing required option -input
Usage: $HADOOP_HOME/bin/hadoop [--config dir] jar \
$HADOOP_HOME/hadoop-streaming.jar [options]
Options:
-input <path> DFS input file(s) for the Map step
-output <path> DFS output directory for the Reduce step
-mapper <cmd|JavaClassName> The streaming command to run
-combiner <JavaClassName> Combiner has to be a Java class
-reducer <cmd|JavaClassName> The streaming command to run
-file <file> File/dir to be shipped in the Job jar file
-dfs <h:p>|local Optional. Override DFS configuration
-jt <h:p>|local Optional. Override JobTracker configuration
-additionalconfspec specfile Optional.
-inputformat TextInputFormat(default)|SequenceFileAsTextInputFormat|JavaClassName Optional.
-outputformat TextOutputFormat(default)|JavaClassName Optional.
-partitioner JavaClassName Optional.
-numReduceTasks <num> Optional.
-inputreader <spec> Optional.
-jobconf <n>=<v> Optional. Add or override a JobConf property
-cmdenv <n>=<v> Optional. Pass env.var to streaming commands
-mapdebug <path> Optional. To run this script when a map task fails
-reducedebug <path> Optional. To run this script when a reduce task fails
-cacheFile fileNameURI
-cacheArchive fileNameURI
-verbose
]]>
濡傛灉鍦ㄥ紑鍙戜腑 璇蜂嬌鐢?cascading groupby, 榪涜 hadoop join,
鏈枃鍙槸涓烘帰璁ㄥ紕鎳?cascading 瀹炵幇鍋氬噯澶囥?br />
褰撶劧 濡傛灉鏈夋湁浜?hadoop join 榪?璇瘋仈緋繪垜錛屽ぇ瀹朵氦嫻佷笅 錛?br />
鏂囦歡鍙兘闇瑕佺殑涓浜涘弬鑰冿細
hadoop jython ( windows )
jython 錛宩ython 緙栬瘧浠ュ強jar 鍖?
灝戦噺 linux shell
鏈枃浠嬬粛 hadoop 鍙兘浣跨敤鍒扮殑 join 鎺ュ彛嫻嬭瘯 錛屽凡緇忓弬鑰冿細
浣跨敤Hadoop瀹炵幇Inner Join鎿嶄綔鐨勬柟娉曘恌rom娣樺疂銆?/strong>錛歨ttp://labs.chinamobile.com/groups/58_547
涓嬮潰 嫻嬭瘯鍚?錛屾垜榪欏ぇ浣撲笂 瀵?hadoop join 鐨勬柟寮忔槸榪欐牱鐞嗚В鐨?錛堢寽鎯籌級錛?br /> 鏁版嵁1 ; 鏁版嵁2
job1.map( 鏁版嵁1 ) =錛堜復鏃舵枃浠?錛?gt; 鏂囦歡鏍囩ず1+闇瑕乯oin鍒?nbsp; 鏁版嵁
job2.map( 鏁版嵁2 ) =錛堜復鏃舵枃浠?錛?gt; 鏂囦歡鏍囩ず2+闇瑕乯oin鍒?nbsp; 鏁版嵁
涓存椂鏂囦歡 mapred.join.expr 鐢熸垚
job3.map ->
鏂囦歡鏍囩ず1+闇瑕乯oin鍒?: 鏁版嵁
鏂囦歡鏍囩ず2+闇瑕乯oin鍒?: 鏁版嵁
......
job3.Combiner - >
闇瑕乯oin鍒?: 鏂囦歡鏍囩ず1+鏁版嵁
闇瑕乯oin鍒?: 鏂囦歡鏍囩ず2+鏁版嵁
job3.Reducer->
闇瑕乯oin鍒?: 浣跨敤 java-list > 鐢熸垚
鏂囦歡2-鍒梮 [ 鏁版嵁,鏁版嵁... ]
鏂囦歡1-鍒梮 [ 鏁版嵁,鏁版嵁... ]
鐒跺悗 浣犺繖 left join ,鎴?inner join 鎴?xxx join 閫昏緫 灝辮嚜宸辨潵鍚?br />
緇撴灉闆嗗悎
[root@localhost python]# cat /home/megajobs/del/jobs/tools/hadoop-0.18.3/data/090907/1
1
2
3
4
5
[root@localhost python]# cat /home/megajobs/del/jobs/tools/hadoop-0.18.3/data/090907/2
2
4
3
1
淇敼 ..../hadoop-0.18.3/src/examples/python/compile
#!/usr/bin/env bash
export HADOOP_HOME=/home/xx/del/jobs/tools/hadoop-0.18.3
export CASCADING_HOME=/home/xx/del/jobs/tools/cascading-1.0.16-hadoop-0.18.3
export JYTHON_HOME=/home/xx/del/jobs/tools/jython2.2.1
export CLASSPATH="$HADOOP_HOME/hadoop-0.18.3-core.jar"
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# add libs to CLASSPATH
for f in $HADOOP_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $HADOOP_HOME/lib/jetty-ext/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $CASCADING_HOME/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $CASCADING_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $JYTHON_HOME/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# restore ordinary behaviour
unset IFS
/home/xx/del/jobs/tools/jython2.2.1/jythonc -p org.apache.hadoop.examples -d -j $1.jar -c $1.py
/home/xx/del/jobs/tools/hadoop-0.18.3/bin/hadoop jar $1.jar $2 $3 $4 $5 $6 $7 $8 $9
export HADOOP_HOME=/home/xx/del/jobs/tools/hadoop-0.18.3
export CASCADING_HOME=/home/xx/del/jobs/tools/cascading-1.0.16-hadoop-0.18.3
export JYTHON_HOME=/home/xx/del/jobs/tools/jython2.2.1
export CLASSPATH="$HADOOP_HOME/hadoop-0.18.3-core.jar"
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# add libs to CLASSPATH
for f in $HADOOP_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $HADOOP_HOME/lib/jetty-ext/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $CASCADING_HOME/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $CASCADING_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $JYTHON_HOME/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# restore ordinary behaviour
unset IFS
/home/xx/del/jobs/tools/jython2.2.1/jythonc -p org.apache.hadoop.examples -d -j $1.jar -c $1.py
/home/xx/del/jobs/tools/hadoop-0.18.3/bin/hadoop jar $1.jar $2 $3 $4 $5 $6 $7 $8 $9
綆鍗?鏁版嵁 閾炬帴 :
from org.apache.hadoop.fs import Path
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred.lib import *
from org.apache.hadoop.mapred.join import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class tMap(Mapper, MapReduceBase):
def map(self, key, value, output, reporter):
output.collect( Text( str(key) ) , Text( value.toString() ))
def main(args):
conf = JobConf(tMap)
conf.setJobName("wordcount")
conf.setMapperClass( tMap )
FileInputFormat.setInputPaths(conf,[ Path(sp) for sp in args[1:-1]])
conf.setOutputKeyClass( Text )
conf.setOutputValueClass( Text )
conf.setOutputPath(Path(args[-1]))
JobClient.runJob(conf)
if __name__ == "__main__":main(sys.argv)
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred.lib import *
from org.apache.hadoop.mapred.join import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class tMap(Mapper, MapReduceBase):
def map(self, key, value, output, reporter):
output.collect( Text( str(key) ) , Text( value.toString() ))
def main(args):
conf = JobConf(tMap)
conf.setJobName("wordcount")
conf.setMapperClass( tMap )
FileInputFormat.setInputPaths(conf,[ Path(sp) for sp in args[1:-1]])
conf.setOutputKeyClass( Text )
conf.setOutputValueClass( Text )
conf.setOutputPath(Path(args[-1]))
JobClient.runJob(conf)
if __name__ == "__main__":main(sys.argv)
榪愯
./compile test file:///home/xx/del/jobs/tools/hadoop-0.18.3/data/090907/1 file:///home/xx/del/jobs/tools/hadoop-0.18.3/data/090907/2 file:///home/xx/del/jobs/tools/hadoop-0.18.3/tmp/wc78
緇撴灉:
[xx@localhost wc78]$ cat ../wc78/part-00000
0 1
0 2
2 4
2 2
4 3
4 3
6 1
6 4
8 5
綆鍗曠殑鏁版嵁 join :
from org.apache.hadoop.fs import Path
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred.lib import *
from org.apache.hadoop.mapred.join import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class tMap(Mapper, MapReduceBase):
def map(self, key, value, output, reporter):
output.collect( Text( str(key) ) , Text( value.toString() ))
def main(args):
conf = JobConf(tMap)
conf.setJobName("wordcount")
conf.setMapperClass( tMap )
conf.set("mapred.join.expr", CompositeInputFormat.compose("override",TextInputFormat, args[1:-1] ) )
conf.setOutputKeyClass( Text )
conf.setOutputValueClass( Text )
conf.setInputFormat(CompositeInputFormat)
conf.setOutputPath(Path(args[-1]))
JobClient.runJob(conf)
if __name__ == "__main__":main(sys.argv)
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred.lib import *
from org.apache.hadoop.mapred.join import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class tMap(Mapper, MapReduceBase):
def map(self, key, value, output, reporter):
output.collect( Text( str(key) ) , Text( value.toString() ))
def main(args):
conf = JobConf(tMap)
conf.setJobName("wordcount")
conf.setMapperClass( tMap )
conf.set("mapred.join.expr", CompositeInputFormat.compose("override",TextInputFormat, args[1:-1] ) )
conf.setOutputKeyClass( Text )
conf.setOutputValueClass( Text )
conf.setInputFormat(CompositeInputFormat)
conf.setOutputPath(Path(args[-1]))
JobClient.runJob(conf)
if __name__ == "__main__":main(sys.argv)
榪愯緇撴灉 ( ) :
./compile test file:///home/xx/del/jobs/tools/hadoop-0.18.3/data/090907/1 file:///home/xx/del/jobs/tools/hadoop-0.18.3/data/090907/2 file:///home/xx/del/jobs/tools/hadoop-0.18.3/tmp/wc79
[xx@localhost wc78]$ cat ../wc79/part-00000
0 2
2 4
4 3
6 1
8 5
]]>
榪欐 鍦?鑷繁鐢佃剳涓?nbsp; 緇堜簬 瀹屾垚,涓嬮潰浠嬬粛榪囩▼:
嫻嬭瘯鐜錛?br /> 渚濈劧鐨?windows + cygwin
hadoop 0.18 # C:/cygwin/home/lky/tools/java/hadoop-0.18.3
jython 2.2.1 # C:/jython2.2.1
鍙傝? PythonWordCount
鍚姩 hadoop 騫跺埌 hdoop_home 涓?br />
# 鍦ㄤ簯鐜涓垱寤?input 鐩綍
$>bin/hadoop dfs -mkdir input
# 鍦?鍖?hadoop 鐨?NOTICE.txt 鎷瘋礉鍒?input 鐩綍涓?/strong>
$>bin/hadoop dfs -copyFromLocal c:/cygwin/home/lky/tools/java/hadoop-0.18.3/NOTICE.txt hdfs:///user/lky/input
$>cd src/examples/python
# 鍒涘緩 涓?鑴氭湰 ( jy->jar->hd run ) 涓姝ュ畬鎴?
# 褰撶劧 鍦?linux 鍐欎釜鑴氭湰姣旇繖 濂界湅 鍛靛懙錛?br /> $>vim run.bat
# 淇敼 jythonc 鎵撳寘 鐜 銆?+hadoop jar
$>vim C:\jython2.2.1\Tools\jythonc\jythonc.py
# 榪愯
C:/cygwin/home/lky/tools/java/hadoop-0.18.3/src/examples/python>
run.bat WordCount.py hdfs:///user/lky/input file:///c:/cygwin/home/lky/tools/java/hadoop-0.18.3/tmp2
$>bin/hadoop dfs -mkdir input
# 鍦?鍖?hadoop 鐨?NOTICE.txt 鎷瘋礉鍒?input 鐩綍涓?/strong>
$>bin/hadoop dfs -copyFromLocal c:/cygwin/home/lky/tools/java/hadoop-0.18.3/NOTICE.txt hdfs:///user/lky/input
$>cd src/examples/python
# 鍒涘緩 涓?鑴氭湰 ( jy->jar->hd run ) 涓姝ュ畬鎴?
# 褰撶劧 鍦?linux 鍐欎釜鑴氭湰姣旇繖 濂界湅 鍛靛懙錛?br /> $>vim run.bat
"C:\Program Files\Java\jdk1.6.0_11\bin\java.exe" -classpath "C:\jython2.2.1\jython.jar;%CLASSPATH%" org.python.util.jython C:\jython2.2.1\Tools\jythonc\jythonc.py -p org.apache.hadoop.examples -d -j wc.jar -c %1
sh C:\cygwin\home\lky\tools\java\hadoop-0.18.3\bin\hadoop jar wc.jar %2 %3 %4 %5 %6 %7 %8 %9
sh C:\cygwin\home\lky\tools\java\hadoop-0.18.3\bin\hadoop jar wc.jar %2 %3 %4 %5 %6 %7 %8 %9
# 淇敼 jythonc 鎵撳寘 鐜 銆?+hadoop jar
$>vim C:\jython2.2.1\Tools\jythonc\jythonc.py
# Copyright (c) Corporation for National Research Initiatives
# Driver script for jythonc2. See module main.py for details
import sys,os,glob
for fn in glob.glob('c:/cygwin/home/lky/tools/java/hadoop-0.18.3/*.jar') :sys.path.append(fn)
for fn in glob.glob('c:/jython2.2.1/*.jar') :sys.path.append(fn)
for fn in glob.glob('c:/cygwin/home/lky/tools/java/hadoop-0.18.3/lib/*.jar') :sys.path.append(fn)
import main
main.main()
import os
os._exit(0)
# Driver script for jythonc2. See module main.py for details
import sys,os,glob
for fn in glob.glob('c:/cygwin/home/lky/tools/java/hadoop-0.18.3/*.jar') :sys.path.append(fn)
for fn in glob.glob('c:/jython2.2.1/*.jar') :sys.path.append(fn)
for fn in glob.glob('c:/cygwin/home/lky/tools/java/hadoop-0.18.3/lib/*.jar') :sys.path.append(fn)
import main
main.main()
import os
os._exit(0)
# 榪愯
C:/cygwin/home/lky/tools/java/hadoop-0.18.3/src/examples/python>
run.bat WordCount.py hdfs:///user/lky/input file:///c:/cygwin/home/lky/tools/java/hadoop-0.18.3/tmp2
緇撴灉杈撳嚭錛?/strong>
cat c:/cygwin/home/lky/tools/java/hadoop-0.18.3/tmp2/part-00000
(http://www.apache.org/). 1
Apache 1
Foundation 1
Software 1
The 1
This 1
by 1
developed 1
includes 1
product 1
software 1
涓嬮潰閲嶅ご鏉ヤ簡 錛氾紙綆媧佺殑 jy hdoop 浠ｇ爜錛?/strong>
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from org.apache.hadoop.fs import Path
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class WordCountMap(Mapper, MapReduceBase):
one = IntWritable(1)
def map(self, key, value, output, reporter):
for w in value.toString().split():
output.collect(Text(w), self.one)
class Summer(Reducer, MapReduceBase):
def reduce(self, key, values, output, reporter):
sum = 0
while values.hasNext():
sum += values.next().get()
output.collect(key, IntWritable(sum))
def printUsage(code):
print "wordcount [-m <maps>] [-r <reduces>] <input> <output>"
sys.exit(code)
def main(args):
conf = JobConf(WordCountMap);
conf.setJobName("wordcount");
conf.setOutputKeyClass(Text);
conf.setOutputValueClass(IntWritable);
conf.setMapperClass(WordCountMap);
conf.setCombinerClass(Summer);
conf.setReducerClass(Summer);
try:
flags, other_args = getopt.getopt(args[1:], "m:r:")
except getopt.GetoptError:
printUsage(1)
if len(other_args) != 2:
printUsage(1)
for f,v in flags:
if f == "-m":
conf.setNumMapTasks(int(v))
elif f == "-r":
conf.setNumReduceTasks(int(v))
conf.setInputPath(Path(other_args[0]))
conf.setOutputPath(Path(other_args[1]))
JobClient.runJob(conf);
if __name__ == "__main__":
main(sys.argv)
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from org.apache.hadoop.fs import Path
from org.apache.hadoop.io import *
from org.apache.hadoop.mapred import *
import sys
import getopt
class WordCountMap(Mapper, MapReduceBase):
one = IntWritable(1)
def map(self, key, value, output, reporter):
for w in value.toString().split():
output.collect(Text(w), self.one)
class Summer(Reducer, MapReduceBase):
def reduce(self, key, values, output, reporter):
sum = 0
while values.hasNext():
sum += values.next().get()
output.collect(key, IntWritable(sum))
def printUsage(code):
print "wordcount [-m <maps>] [-r <reduces>] <input> <output>"
sys.exit(code)
def main(args):
conf = JobConf(WordCountMap);
conf.setJobName("wordcount");
conf.setOutputKeyClass(Text);
conf.setOutputValueClass(IntWritable);
conf.setMapperClass(WordCountMap);
conf.setCombinerClass(Summer);
conf.setReducerClass(Summer);
try:
flags, other_args = getopt.getopt(args[1:], "m:r:")
except getopt.GetoptError:
printUsage(1)
if len(other_args) != 2:
printUsage(1)
for f,v in flags:
if f == "-m":
conf.setNumMapTasks(int(v))
elif f == "-r":
conf.setNumReduceTasks(int(v))
conf.setInputPath(Path(other_args[0]))
conf.setOutputPath(Path(other_args[1]))
JobClient.runJob(conf);
if __name__ == "__main__":
main(sys.argv)
]]>