golang--2016gopher北京大会(2)

三、七牛老许

qlang: github qiniu/qlang

microservice architecture: http://martinfowler.com/articles/microservices.html

Service Governance

• Authorization
• Logging
• Change management
• Central configuration
• Scale in and scale out
• Overload Protection
• Service degradation
• Monitor performance and health
• Manage how and by whom services are used 
• Topology discovery and failure recovery

Overload Protection

• N = Alert threshold
• Important things
• keep limit to N*2,not N
• kill slow requests(SLA)

 

四、Dave Cheney 澳洲golang社区负责人，运行悉尼go用户组

http://dave.cheney.net/

Further reading: runtime/pprof

Further reading: Google Perf Tools

• pprof

-memprofilerate=N adjusts the profile rate to 1/N

#visualise mem profile
go build -gcflags='-memprofile=/tmp/m.p'
go tool pprof --alloc_objects -svg $(go tool -n compile) /tmp/m.p > alloc_objects.svg
go tool pprof --inuse_objects -svg $(go tool -n compile) /tmp/m.p > alloc_objects.svg

•  benchmarking

#fib.go:
func Fib(n int) int {
    if n<2 {
        return n
    }
    return Fib(n-1)+Fib(n-2)
}
#fib_test.go:
import "testing"
func BenchmarkFib(b *testing.B) {
    for n:=0;n<b.N;n++ {
        Fib(20)
    }
}
#go test -bench=. ./fib

• • do benchmark multiple times or use "-count=" flag
  • tools like rsc.io/benchstat are useful for comparing results

go test -bench=. -count=5 ./fib> old.txt 
go test -bench=. -count=5 ./fib |tee new.txt 
benchstat old.txt new.txt
benchstat {old,new}.txt

• •  how fast will this bench? I think it's super fast! 

package popcnt

import "testing"

const m1 = 0x5555555555
const m2 = 0x33333
const m4 = 0x0f0f0f
const h01 = 0x0101010101

func popcnt(x uint64) uint64 {
	x -= (x >> 1) & m1
	x = (x & m2) + ((x >> 2) & m2)
	x = (x + (x >> 4)) & m4
	return (x * h01) >> 56
}
func benchmarkPopcnt(b *testing.B) {
	for i := 0; i < b.N; i++ {
		popcnt(uint64(i)) //optimied away
	}
}

#go test -bench=. ./popcnt

• • 但是被优化没有效果，要改为如下才有bench的效果

#change it to this:
var result uint64
func benchmarkPopcnt(b *testing.B) {
    var r uint64
    for i:=0;i<b.N;i++ {
        r = popcnt(uint64(i))
    }
    result=r
}

• • 自己测试还是没效果,不知道为什么
•  CPU profile

go test -run=XXX -bench=IndexByte -cpuprofile=/tmp/c.p bytes
go tool pprof bytes.test /tmp/c.p
#note: -run=XXX to disable test,you only need benchmark

• godoc

cd godoc
golang.org/x/tools/cmd/godoc
vim main.go
   add one line in main--defer profile.Start().Stop()
   then exec in vim:GoImports
go install -v .
godoc -http=9000
open browser and open localhost:9000
go tool pprof $(which godoc) /var/...cpu.pprof

•  enable output of GCloggin

env GODEBUG=gctrace=1 godoc -http=:8080

• import net/http/pprof package will register a handler at /debug/pprof and default http.serveMux

it will visible if you use http.listenSndServe(adress,nil)

godoc -http:=8080,show /debug/pprof

•  some slice and string translate...

五、晚场辩论

rust-lang.org

//GO
err := f(x)
if err != nil {
    retrun err;
}

//Rust
try!(f(x));
f(x)?

 一个关于冗长与magic的讨论

六、原射手CEO

gomobile和动态库的使用

 

七、米嘉

github.com/mijia/gobuildweb

github.com/mijia/web-starter-kit

@fswatch $(GO_FILES) $(TEMPLATES) | xargs -n1 -l{} make restart ||make kill

• chubby--at the heart of google
• go Proverbs: simple,Poetic,Pithy,no magic,composing,interface,adaptor
• request --> onion --> response
• basiclly we put a lot of things inside handler: like  parse request,query db,user auth, CRUD model objects, biz logic, then render the result to response xml,json etc. or panic/error.

• go web packages:

dozens of frameworks

julienschmidt/httprouter

codegangstar/negroni

www.gorillatoolkit.org

• go generate according to database scheme

github.com/mijia/modelg 

• go benchmark

load_test.go//测试1000000LRUcache

package demo1

import (
    "fmt"
    "testing"
    "time"
)

//TestPutPerf load tests
func TestPutPerf(t *testing.T) {
    fmt.Printf("load testing on 1M size cache\n")
    size := 1000000
    cache := newLRUCache(size)
    start := time.Now()
    for i,td := range testData {
        if i%10000 == 0 {
            end := time.Now()
            fmt.Printf("Num: %d, Used time: %v\n", i, end.Sub(start))
            start = end
        }
        cache.Put(td.url,td.body)
    }
}

 

go test -c
./demo1.test -test.run=Perf

//lru_cache_test.go
package demo1
import (
    "fmt"
    "testing"
)

type testDataType struct {
    url string
    body string
}

var testData []testDataType

const testDataSize = 10000100

func init () {
    testData = make([]testDataType, testDataSize)
    for i:= 0; i< testDataSize; i++ {
        url := fmt.Sprintf("www.fake.com/%d",i)
       testData[i] = testDataType {
            url: url,
            body: fmt.Sprintf("This is page <b>%s</b>!",url),
       }
    }
}

func TestBasics(t *testing.T) {
    cache := newLRUCache(2)
    cache.Put("1","1")
    cache.Put("2","2")
    res, ok := cache.Get("1")
    if !ok := || res != "1" {
        t.Errorf("unexpected!")
    }
    cache.Put("4","4")
    if _, ok := cache.Get("2");ok {
        t.Errorf("unexpected")
    }
}

//go tool pprof
func BenchmarkCacheFull1KSize(b *test.B) {
    benchmarkCacheFull(b,1000)
}
func BenchmarkCacheFull1MSize(b *testing.B) {
    benchmarkCacheFull(b,1000000)
}
func benchmarkCacheFull(b *testing.B, size int) {
    cache := newLRUCache(size)
    for i :=0;i<size;i++ {
        cache.Put(testData[i].url,testData[i].body)
    }
//两个Put之间有noise必须先做profiling,
//go tool pprof demo1.test prof.out -->#web
    f, _ := os.Create("prof.out")
    pprof.StartCPUProfile(f)
    defer pprof.StopCPUProfile()
//先填满，之后测试每push 一个item需要多长时间
    b.ResetTimer()
    for i :=0; i< b.N; i++ {
        c:=(i+size) % testDataSize
        cache.Put(testData[c].url, testData[c].body)
    }
}

//demo2
//go test -v . -run XXX -bench Mill
//./demo2.test -test.run XXX -test.bench Milli -test.cpuprofile=prof.out
//go tool pprof demo2.test prof.out --> web

 

./demo1.test -test.run=xxx -test.bench Full
#./demo1.test -test.run=xxx -test.bench Full -test.cpuprofile=prof.out 这是不准确的，因为有很多noise,用上面方法

 

 

八、Marcel van Lohuizen -- Google Go team

golang.org/x/text 实现

• I18n and L10n 国际化与本地化

搜索排序，大小写，双向文本，注入翻译，数字货币，日期，单位转换

• Go uses UTF-8

const beijing="北京市"
for index,runeValue :=range beijing {
fmt.Printf("%#U 从第%d字节开始\n",runeValue,index)
}
//北从0，京从3，市从6

• 文本的序列本质

const flags="🇪🇭🇹🇻" //国家代码mc+nl
fmt.Println(flags[4:])

• Transformer 接口

type Transformer interface {
    Transform(dst,src []byte, atEOF bool) (nDst, nSrc int, err error)
    Reset()
}

• Normalization 以NFC格式向w写入文本流

norm.NFC.Writer(w) 

• 不同语言不同大小写算法

toTitle:=cases.Title(language.Dutch)
fmt.Println(toTitle.String("'n ijsberg"))
//'n IJsberg

•  多语言搜索与排序

e<ℨ<f

ª==aa

search

m:search.New(language.Danish,search.IgnoreCase,search.IgnoreDiacritics)
start,end:=m.IndexString(text,s)
match:=s[start:end]

•  collate Example

import (
"fmt"
"golang.org/x/text/collate"
"golang.org/x/text/language"
)

func main () {
  a:=[]string("北京市","上海市","广州市")
  for _,tag := range []string{"en","zh","zh-u-co-stroke"} {
    collate.New(language.Make(tag)).SortStrings(a)
    fmt.Println(a)
  }
}

// 上 北 广；北广上；上广北

 语言匹配

import (
 "http"
 "golang.org/x/text/language"
)

var matcher = language.NewMatcher([]language.Tag{
    language.SimplifiedChinese,
    language.AmericanEnglish,
})

func handle(w http.ResponseWriter, r* http.Request){
    prefs,_,_:=language.ParseAcceptLanguage(r.Header.Get("Accept-Languge"))

    tag,_,_:=matcher.Match(prefs...)
    //use tag: it includes carried over user preference
}

 在代码中插入翻译结果

import "golang.org/x/text/message"
message.SetString(language.Dutch,"%s went to %s","%s is in %s geweest.")
message.SetString(language.SimplifiedChinese,"%s went to %s","%s 去了 %s ")

 

• 参考

godoc.org/golang.org/x/text

blog.golang.org/matchlang

blog.golang.org/normalization

blog.golang.org/strings

golang.org/issue

 

九、毛剑--bilibili

bfs分布式小文件存储: github.com/Terry-Mao/bfs

基于facebook Haystack Paper 

• 核心：proxy、directory、pitchfork、store
• 运维：ops
• 依赖：Hbase用户数据 zookeeper元数据
• 流量走向 proxy->directory->store

 

十、李炳毅--百度

if r<= 0x7F {
    switch 
    case '0' <= r && r <= '9':
        return false
    case 'a' <= r && r <= 'z':
        return false
    case 'A' <= r && r <= 'Z':
        return false
    case ==
        return false
    return true
}
if unicode.IsLetter(r) || unicode.IsDigit(r) {
    return false  
}

 

十一、TiDB PingCAP创始人

• 1.interface 性能问题

var val interface{}
val = int64(10)

把整形赋值给interface会触法一次内存分配，会有一个数量级的开销

var d Datum
d.SetInt64(10)

换成struct提升10倍以上

• 2.Protobuf Go默认实现性能低，推荐gogo替代
• 3.跨数据中心复制，Raft(Port from etcd) 作者Diego Ongaro。相对multi－paxos简单成熟稳定

 

十二、陈辉－人工智能初创公司

• 1.悟空搜索：github.com/huichen/wukong

主协程，用于收发用户请求。分词器协程，负责分词。索引器协程，负责查找索引。排序器。

package  main
import (
"github.com/huichen/wukong/engine"
"github.com/huichen/wukong/types"
"log"
)

var(
//searcher是协程安全的
searcher = engine.Engine{}
)

func main() {
searcher.Init(types.EngineInitOptions{SegmentedDictionaries:"github.com/huichen/wukong/data/dictionary.txt"})
defer searcher.Close()

searcher.IndexDocument(0,types.DocumentIndexData{Content: " XXXXX"})
searcher.IndexDocument(1,types.DocumentIndexData{Content: " XXXXX"})
searcher.IndexDocument(2,types.DocumentIndexData{Content: " XXXXX"})

searcher.FlushIndex()

log.Print(searcher.Search(types.SearchRequest{Text:"XX"}))

}

 基于悟空搜索的计算技术 AHA: Ad Hoc Analytics引擎

github.com/huichen/aha

• 2.sego分词：huichen/sego

package main

import (
"fmt"
"github.com/huichen/sego"
)

func main() {
//dictionary
var segmenter sego.Segmenter
segmenter.LoadDictionary("github.com/huichen/sego/data/dictionary.txt")

//分词
text := []byte("北京市东城区XXXX")
segments := segmenter.Segment(text)

//处理分词结果
fmt.Println(sego.SegmentsToString(segments,false))

}

 Double-Array Trie实现是一般Trie两倍效率

github.com/adamzy/cedar-go

•  3.弥勒佛机器学习：huichen/mlf

监督式学习：最大熵分类模型，决策树模型

非监督式学习：聚类问题

在线学习：在线梯度递降模型

神经网络：Restricted Boltzmann machine

var (
libsvm_file = flag.String("input","","libsvm格式数据文件，训练数据")
model = flag.String("model","model.mlf","写入的模型文件")

//机器学习参数
learning_rate = flag.Float64("learning_rate",0.01,"学习率")
batch_size = flag.Int("batch_size",100,"梯度递降法的batch尺寸：1为stochasitic，其它值为mini batch")
delta = flag.Float64("delta",1e-4,"权重变化量和权重比值，dw/w 的绝对值，小于此值为收敛")
maxIter = flag.Int("max_iter",0,"优化器最多迭代多少次")
hidden = flag.Int("hidden",10,"多少个隐藏单元")
numCD = flag.Int("cd",1,"CD次数")
useBinary = flag.Bool("binary_hidden",true,"是否使用抽样隐藏单元")
)

 

代码over，郑重声明：部分代码已测试, 但以上代码均不保证可运行，不保证无错误，并仅用于总结演讲ppt。