{"id":54,"date":"2023-12-08T08:48:00","date_gmt":"2023-12-08T08:48:00","guid":{"rendered":"https:\/\/tensor.agenthub.uk\/?p=54"},"modified":"2024-05-13T03:46:48","modified_gmt":"2024-05-13T03:46:48","slug":"llama","status":"publish","type":"post","link":"https:\/\/tensorzen.blog\/?p=54","title":{"rendered":"Llama \u91cd\u5199\u65e5\u5fd7[\u672a\u5b8c…]"},"content":{"rendered":"\n

\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u540e\u62a5\u9519<\/p>\n\n\n\n

<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
from<\/span> llama <\/span>import<\/span> tokenizer<\/span>,<\/span> Llama<\/span>,<\/span> Dialog<\/span><\/span>\ncheckpoint_dir <\/span>=<\/span> <\/span>"<\/span>\/training-data\/pakcages\/llama\/llama-2-7b-chat<\/span>"<\/span><\/span>\ntokenizer_path <\/span>=<\/span> <\/span>"<\/span>\/training-data\/pakcages\/llama\/tokenizer.model<\/span>"<\/span><\/span>\ntemperature <\/span>=<\/span> <\/span>0.75<\/span><\/span>\ntop_p <\/span>=<\/span> <\/span>0.9<\/span><\/span>\nmax_seq_len <\/span>=<\/span> <\/span>128<\/span><\/span>\nmax_gen_len <\/span>=<\/span> <\/span>64<\/span><\/span>\nmax_batch_size <\/span>=<\/span> <\/span>4<\/span><\/span>\n<\/span>\ngenerator <\/span>=<\/span> Llama<\/span>.<\/span>build<\/span>(<\/span><\/span>\n    <\/span>ckpt_dir<\/span>=<\/span>checkpoint_dir<\/span>,<\/span><\/span>\n    <\/span>tokenizer_path<\/span>=<\/span>tokenizer_path<\/span>,<\/span><\/span>\n    <\/span>max_seq_len<\/span>=<\/span>max_seq_len<\/span>,<\/span><\/span>\n    <\/span>max_batch_size<\/span>=<\/span>max_batch_size<\/span>)<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
ValueError: Error initializing torch.distributed using env:\/\/ rendezvous: environment variable RANK expected, but not set<\/p>\n\n\n\n
\u662f\u6e90\u7801\u91cc\u9762\u8fd9\u4e00\u6bb5\u5f15\u8d77\u7684\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
if<\/span> <\/span>not<\/span> torch<\/span>.<\/span>distributed<\/span>.<\/span>is_initialized<\/span>():<\/span><\/span>\n    torch<\/span>.<\/span>distributed<\/span>.<\/span>init_process_group<\/span>(<\/span>"<\/span>nccl<\/span>"<\/span>)<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u542f\u52a8\u4e0d\u8d77\u6765\u770b\u6837\u5b50\u662f\u56e0\u4e3a\u5206\u5e03\u5f0f\u7684\u95ee\u9898\u3002\u6211\u5c1d\u8bd5\u7ed5\u5f00\u5206\u5e03\u5f0f\uff0c\u4ece\u5b83\u7684build\u51fd\u6570\u5f00\u59cb\u770b\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
checkpoints <\/span>=<\/span> <\/span>sorted<\/span>(<\/span>Path<\/span>(<\/span>ckpt_dir<\/span>).<\/span>glob<\/span>(<\/span>"<\/span>*.pth<\/span>"<\/span>))<\/span><\/span>\nckpt_path <\/span>=<\/span> checkpoints<\/span>[<\/span>get_model_parallel_rank<\/span>()]<\/span><\/span>\ncheckpoint <\/span>=<\/span> torch<\/span>.<\/span>load<\/span>(<\/span>ckpt_path<\/span>,<\/span> <\/span>map_location<\/span>=<\/span>"<\/span>cpu<\/span>"<\/span>)<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u68c0\u67e5\u7ed9\u5b9a\u7684checkpoint_dir\u662f\u5426\u5305\u542bpth\u6587\u4ef6\uff0c7B\u7684\u6a21\u578b\u53ea\u6709\u4e00\u4e2apth\u6587\u4ef6\uff0c\u6240\u4ee5\u4e00\u4e2a\u8fdb\u7a0b\u5c31\u53ef\u4ee5\u4e86\uff0c\u6211\u60f3get_model_parallel_rank()\u5927\u6982\u610f\u601d\u5373\u662f\u6709\u51e0\u4e2a\u6587\u4ef6\u5c31\u542f\u52a8\u591a\u5c11\u4e2a\u8fdb\u7a0b\uff0c\u4ee3\u7801\u6765\u81eafacebook\u56e2\u961f\u5f00\u53d1\u5e76\u884c\u8bad\u7ec3\u5305fairscale:<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
def<\/span> <\/span>get_model_parallel_rank<\/span>()<\/span> <\/span>-><\/span> <\/span>int<\/span>:<\/span><\/span>\n    <\/span>"""<\/span>Return my rank for the model parallel group.<\/span>"""<\/span><\/span>\n    <\/span>return<\/span> torch<\/span>.<\/span>distributed<\/span>.<\/span>get_rank<\/span>(<\/span>group<\/span>=<\/span>get_model_parallel_group<\/span>())<\/span><\/span>\n<\/span>\ndef<\/span> <\/span>get_model_parallel_group<\/span>()<\/span> <\/span>-><\/span> torch<\/span>.<\/span>distributed<\/span>.<\/span>ProcessGroup<\/span>:<\/span><\/span>\n    <\/span>"""<\/span>Get the model parallel group the caller rank belongs to.<\/span>"""<\/span><\/span>\n    <\/span>assert<\/span> _MODEL_PARALLEL_GROUP <\/span>is<\/span> <\/span>not<\/span> <\/span>None<\/span>,<\/span> <\/span>"<\/span>model parallel group is not initialized<\/span>"<\/span><\/span>\n    <\/span>return<\/span> _MODEL_PARALLEL_GROUP<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u54b1\u4e0d\u7ba1\u8fd9\u4e9b\uff0c7B\u53cd\u6b63\u5c31\u4e00\u4e2a\u53c2\u6570\u6587\u4ef6\uff0c\u76f4\u63a5\u4ece\u6587\u4ef6\u5939\u52a0\u8f7d\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
from<\/span> pathlib <\/span>import<\/span> Path<\/span><\/span>\ncheckpoints <\/span>=<\/span> <\/span>sorted<\/span>(<\/span>Path<\/span>(<\/span>checkpoint_dir<\/span>).<\/span>glob<\/span>(<\/span>"<\/span>*.pth<\/span>"<\/span>))<\/span><\/span>\n# Llama-2-7b model weights are distributed in a single file.<\/span><\/span>\ncheckpoint <\/span>=<\/span> checkpoints<\/span>[<\/span>0<\/span>]<\/span><\/span>\ndevice <\/span>=<\/span> torch<\/span>.<\/span>device<\/span>(<\/span>'<\/span>cuda<\/span>'<\/span> <\/span>if<\/span> torch<\/span>.<\/span>cuda<\/span>.<\/span>is_available<\/span>()<\/span> <\/span>else<\/span> <\/span>'<\/span>cpu<\/span>'<\/span>)<\/span><\/span>\ncheckpoint <\/span>=<\/span> torch<\/span>.<\/span>load<\/span>(<\/span>checkpoint<\/span>,<\/span> <\/span>map_location<\/span>=<\/span>device<\/span>)<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
7b\u6a21\u578b\u76f4\u63a5\u52a0\u8f7d\u8fdb\u663e\u5361\u5360\u7528\u4e8613671MB\u7684\u663e\u5b58\uff1a<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
huggingface\u8f6c\u6362<\/h2>\n\n\n\n
\u7b97\u4e86\uff0c\u8fd8\u662f\u5148\u4e0d\u91cd\u5199\u4e86\uff0c\u5148\u7528huggingface\u8f6c\u6362\u5427\uff0c\u5b89\u88c5\u4e00\u4e0btransformers\uff0c\u4ed6\u7684\u8f6c\u6362\u51fd\u6570\u5728src\/transformers\/models\/llama\/convert_llama_weights_to_hf.py \u8fd9\u91cc<\/a>\u53ef\u4ee5\u770b\u6e90\u6587\u4ef6\u3002<\/p>\n\n\n\n
\u672c\u8eab\u5b89\u88c5transformers\u7684\u65f6\u5019\u5df2\u7ecf\u5b89\u88c5\u4e86\u8fd9\u4e2a\u6a21\u5757\uff0c\u5199\u4e2a\u811a\u672c\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
from<\/span> transformers<\/span>.<\/span>models<\/span>.<\/span>llama<\/span>.<\/span>convert_llama_weights_to_hf <\/span>import<\/span> main<\/span><\/span>\n<\/span>\nif<\/span> __name__ <\/span>==<\/span> <\/span>"<\/span>__main__<\/span>"<\/span>:<\/span><\/span>\n    <\/span>main<\/span>()<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u76f4\u63a5\u8fd9\u4e2a\u811a\u672c\u5c31\u884c\u4e86\uff0c\u6267\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
# python convert.py --help<\/span><\/span>\nusage<\/span>:<\/span> convert<\/span>.<\/span>py <\/span>[<\/span>-<\/span>h<\/span>]<\/span> <\/span>[<\/span>--<\/span>input_dir INPUT_DIR<\/span>]<\/span> <\/span>[<\/span>--<\/span>model_size <\/span>{<\/span>7B<\/span>,<\/span>7Bf<\/span>,<\/span>13B<\/span>,<\/span>13Bf<\/span>,<\/span>30B<\/span>,<\/span>34B<\/span>,<\/span>65B<\/span>,<\/span>70B<\/span>,<\/span>70Bf<\/span>,<\/span>tokenizer_only<\/span>}]<\/span> <\/span>[<\/span>--<\/span>output_dir OUTPUT_DIR<\/span>]<\/span> <\/span>[<\/span>--<\/span>safe_serialization SAFE_SERIALIZATION<\/span>]<\/span><\/span>\n<\/span>\noptions<\/span>:<\/span><\/span>\n  <\/span>-<\/span>h<\/span>,<\/span> <\/span>--<\/span>help<\/span>            show this <\/span>help<\/span> message <\/span>and<\/span> <\/span>exit<\/span><\/span>\n  <\/span>--<\/span>input_dir INPUT_DIR<\/span><\/span>\n                        Location of LLaMA weights<\/span>,<\/span> which contains tokenizer<\/span>.<\/span>model <\/span>and<\/span> model folders<\/span><\/span>\n  <\/span>--<\/span>model_size <\/span>{<\/span>7B<\/span>,<\/span>7Bf<\/span>,<\/span>13B<\/span>,<\/span>13Bf<\/span>,<\/span>30B<\/span>,<\/span>34B<\/span>,<\/span>65B<\/span>,<\/span>70B<\/span>,<\/span>70Bf<\/span>,<\/span>tokenizer_only<\/span>}<\/span><\/span>\n                        <\/span>'<\/span>f<\/span>'<\/span> models correspond to the finetuned versions<\/span>,<\/span> <\/span>and<\/span> are specific to the Llama2 official release<\/span>.<\/span> For more details on Llama2<\/span>,<\/span> checkout the original repo<\/span>:<\/span> https<\/span>:<\/span>\/\/<\/span>huggingface<\/span>.<\/span>co<\/span>\/<\/span>meta<\/span>-<\/span>llama<\/span><\/span>\n  <\/span>--<\/span>output_dir OUTPUT_DIR<\/span><\/span>\n                        Location to write HF model <\/span>and<\/span> tokenizer<\/span><\/span>\n  <\/span>--<\/span>safe_serialization SAFE_SERIALIZATION<\/span><\/span>\n                        Whether <\/span>or<\/span> <\/span>not<\/span> to save using <\/span>`safetensors`<\/span>.<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
–input_dir \u5199\u7684\u662fllama\u7684\u6839\u76ee\u5f55<\/p>\n\n\n\n
–model_size \u9009\u62e9\u8981\u8f6c\u6362\u7684\u6a21\u578b\u53c2\u6570\u91cf\uff0c\u8fd9\u4e2a\u6709\u4e2abug\uff0c\u4f60\u53ea\u80fd\u586b\u63d0\u4f9b\u7684\u90a3\u51e0\u4e2a\u540d\u5b57\uff0c\u95ee\u9898\u662fllama\u7684\u76ee\u5f55\u4e0b\u5bf9\u5e94\u7684\u6a21\u578b\u6587\u4ef6\u540d\u662f”llama-2-*b”\u8fd9\u79cd\uff0c\u8f6c\u6362\u811a\u672c\u4f1a\u53bb”input_dir\/*B”\u4e0b\u9762\u53bb\u627e\u6a21\u578b\u6587\u4ef6\uff0c\u6240\u4ee5\u9700\u8981\u7ed9”llama-2-*b”\u6539\u6210”*B”\u540e\u518d\u6267\u884c\u811a\u672c\u3002<\/p>\n\n\n\n
\u8f6c\u6362\u8fc7\u7a0b\u4e2d\u5e76\u4e0d\u9700\u8981GPU\uff0c\u5b8c\u6210\u4e4b\u540e\u7528transformers\u52a0\u8f7d\u5c31\u884c\u4e86\uff0c7B\u7684\u6a21\u578b\u52a0\u8f7d\u5b8c\u540e\u5212\u5206\u4e8626G\u73b0\u5b58<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
\u6267\u884c\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
total_params <\/span>=<\/span> <\/span>sum<\/span>(<\/span>p<\/span>.<\/span>numel<\/span>()<\/span> <\/span>for<\/span> p <\/span>in<\/span> model<\/span>.<\/span>parameters<\/span>())<\/span><\/span>\nprint<\/span>(<\/span>f<\/span>"Total number of parameters: <\/span>{<\/span>total_params<\/span>}<\/span>"<\/span>)<\/span><\/span>\n>>><\/span> Total number of parameters<\/span>:<\/span> <\/span>6607343616<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u6a21\u578b\u7684\u603b\u53c2\u6570\u662f6,607,343,616<\/p>\n\n\n\n
\u7c97\u7565\u8ba1\u7b97\u4e00\u4e0b\uff0c\u91c7\u7528\u5355\u7cbe\u5ea6(single-precision float-point format)\u5b58\u50a8\u8fd9\u4e9b\u53c2\u6570\u7684\u8bdd\u603b\u5171\u8981\u75286607343616 * 32 \/ 8 \/ 1024 \/ 1024 \/ 1024 = 24.6143G\uff0c\u5982\u679c\u91c7\u7528\u534a\u7cbe\u5ea6\u5b58\u50a8\u770b\u4e0b\uff1a<\/p>\n\n\n\n
<\/circle><\/circle><\/circle><\/g><\/svg><\/span><\/path><\/path><\/svg><\/span>
tokenizer <\/span>=<\/span> LlamaTokenizer<\/span>.<\/span>from_pretrained<\/span>(<\/span>'<\/span>llama_hf\/7Bf<\/span>'<\/span>,<\/span> <\/span>torch_dtype<\/span>=<\/span>torch<\/span>.<\/span>float16<\/span>)<\/span><\/span>\nmodel <\/span>=<\/span> LlamaModel<\/span>.<\/span>from_pretrained<\/span>(<\/span>'<\/span>llama_hf\/7Bf<\/span>'<\/span>,<\/span> <\/span>torch_dtype<\/span>=<\/span>torch<\/span>.<\/span>float16<\/span>)<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n
\u653e\u8fdb\u663e\u5361\u7684\u8bdd\u5212\u5206\u4e8613G\u73b0\u5b58<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
\u8fd8\u60f3\u518d\u964d\u4f4e\u663e\u5b58\u5360\u7528\u5c31\u5f97\u4f7f\u7528quantization\u4e86\uff0c\u540e\u9762\u6574\u7406\u4e0b\u3002<\/p>\n\n\n\n
LlamaForCausalLM\u53ef\u4ee5\u7528\u6765\u751f\u6210\u56de\u7b54\uff0c\u9ed8\u8ba4\u7684LlamaModel\u6ca1\u6709\u8fd9\u529f\u80fd<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5f88\u9057\u61be\u91cd\u5199\u5931\u8d25\u4e86\uff0c\u5b98\u65b9inference\u4f7f\u7528\u4e86meta\u4e4b\u524d\u7684\u4e00\u4e2a\u5305fairscale\uff0c\u5f88\u9ebb\u70e6\uff0c\u540e\u9762\u6709\u5927\u6bb5\u7684\u7a7a\u95f2\u65f6\u95f4\u7684\u8bdd\u518d\u6361\u8d77\u6765\u3002<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[5,12],"tags":[],"class_list":["post-54","post","type-post","status-publish","format-standard","hentry","category-coding","category-llm"],"_links":{"self":[{"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/posts\/54","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=54"}],"version-history":[{"count":11,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/posts\/54\/revisions"}],"predecessor-version":[{"id":511,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=\/wp\/v2\/posts\/54\/revisions\/511"}],"wp:attachment":[{"href":"https:\/\/tensorzen.blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=54"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=54"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/tensorzen.blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=54"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}