a3eb24a0fc
* feat: Optimize output of wayid2nodeids format, use delta format to comparess data Issues: https://github.com/Telenav/osrm-backend/issues/31 * feat: Implement logic to compress/decompress file to snappy. issues: https://github.com/Telenav/osrm-backend/issues/31 * feat: Modify osrm speed table generator to support snappy compression format. issues: https://github.com/Telenav/osrm-backend/issues/31 * feat: Fix bug during conversion * feat: Adjust traffic updater's logic to improve performance. * feat: Adjust traffic updater's logic to improve performance. issues: https://github.com/Telenav/osrm-backend/pull/39 * feat: Refine the code for osrm_traffic_updater. issues: https://github.com/Telenav/osrm-backend/issues/31 * fix: fix dead lock in the code. * fix: optimize performance with new architecture. issues: https://github.com/Telenav/osrm-backend/issues/31 * fix: revert way id generator to original solution * fix: Use string to pass between different components issues: https://github.com/Telenav/osrm-backend/issues/31 * fix: update unit test for latest changes issues: https://github.com/Telenav/osrm-backend/issues/31 * fix: remove useless printf * fix: update unit test for osrm_traffic_updater.go * fix: fix the misunderstanding with requirement. Traffic server generates -wayid indicate for traffic flow on reverse direction. issues: https://github.com/Telenav/osrm-backend/issues/31
59 lines
1.1 KiB
Go
59 lines
1.1 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
"github.com/golang/snappy"
|
|
)
|
|
|
|
func loadWay2NodeidsTable(filepath string, sources [TASKNUM]chan string) {
|
|
startTime := time.Now()
|
|
|
|
data := make(chan string)
|
|
go load(filepath, data)
|
|
convert(data, sources)
|
|
|
|
endTime := time.Now()
|
|
fmt.Printf("Processing time for loadWay2NodeidsTable takes %f seconds\n", endTime.Sub(startTime).Seconds())
|
|
}
|
|
|
|
func load(mappingPath string, data chan<- string) {
|
|
defer close(data)
|
|
|
|
f, err := os.Open(mappingPath)
|
|
defer f.Close()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
fmt.Printf("Open idsmapping file of %v failed.\n", mappingPath)
|
|
return
|
|
}
|
|
fmt.Printf("Open idsmapping file of %s succeed.\n", mappingPath)
|
|
|
|
scanner := bufio.NewScanner(snappy.NewReader(f))
|
|
for scanner.Scan() {
|
|
data <- (scanner.Text())
|
|
}
|
|
}
|
|
|
|
|
|
// input data format
|
|
// wayid1, n1, n2
|
|
// wayid2, n3, n4, n5
|
|
func convert(data <-chan string, sources [TASKNUM]chan string) {
|
|
for i := range sources {
|
|
defer close(sources[i])
|
|
}
|
|
|
|
var count int
|
|
for str := range data {
|
|
chanIndex := count % TASKNUM
|
|
count++
|
|
sources[chanIndex] <- str
|
|
}
|
|
}
|
|
|
|
|