Split Cache and HTTP build code into other files

This commit is contained in:
Thomas Schwery 2017-03-07 11:09:35 +01:00
parent 8e5320481a
commit b1113dff25
3 changed files with 271 additions and 192 deletions

90
internals/cache.go Normal file
View file

@ -0,0 +1,90 @@
package internals
import (
"fmt"
"log"
"strconv"
"github.com/syndtr/goleveldb/leveldb"
)
// CacheConn -
type CacheConn struct {
CacheDB *leveldb.DB
}
// SetCache -
func (c *CacheConn) SetCache(cacheDB *leveldb.DB) {
c.CacheDB = cacheDB
}
// GetCachedDataIntSub -
func (c *CacheConn) GetCachedDataIntSub(key int32, subkey string) int32 {
return c.GetCachedDataInt(fmt.Sprintf("%d.%s", key, subkey))
}
// GetCachedDataSub -
func (c *CacheConn) GetCachedDataSub(key int32, subkey string) string {
return c.GetCachedData(fmt.Sprintf("%d.%s", key, subkey))
}
// GetCachedDataIntSub64 -
func (c *CacheConn) GetCachedDataIntSub64(key int64, subkey string) int32 {
return c.GetCachedDataInt(fmt.Sprintf("%d.%s", key, subkey))
}
// GetCachedDataSub64 -
func (c *CacheConn) GetCachedDataSub64(key int64, subkey string) string {
return c.GetCachedData(fmt.Sprintf("%d.%s", key, subkey))
}
// GetCachedDataInt -
func (c *CacheConn) GetCachedDataInt(key string) int32 {
strVal := c.GetCachedData(key)
intVal, convErr := strconv.ParseInt(strVal, 10, 32)
if convErr != nil {
return -1
}
return int32(intVal)
}
// GetCachedData -
func (c *CacheConn) GetCachedData(key string) string {
response, err := c.CacheDB.Get([]byte(key), nil)
if err != nil {
return ""
}
return string(response[:])
}
// PutCacheDataIntSub -
func (c *CacheConn) PutCacheDataIntSub(key int32, subkey string, value int32) {
c.PutCacheDataSub(key, subkey, fmt.Sprintf("%d", value))
}
// PutCacheDataSub -
func (c *CacheConn) PutCacheDataSub(key int32, subkey string, value string) {
c.PutCacheData(fmt.Sprintf("%d.%s", key, subkey), value)
}
// PutCacheDataIntSub64 -
func (c *CacheConn) PutCacheDataIntSub64(key int64, subkey string, value int32) {
c.PutCacheDataSub64(key, subkey, fmt.Sprintf("%d", value))
}
// PutCacheDataSub64 -
func (c *CacheConn) PutCacheDataSub64(key int64, subkey string, value string) {
c.PutCacheData(fmt.Sprintf("%d.%s", key, subkey), value)
}
// PutCacheData -
func (c *CacheConn) PutCacheData(key string, value string) {
err := c.CacheDB.Put([]byte(key), []byte(value), nil)
if err != nil {
log.Fatal(err)
}
}

121
internals/httpclient.go Normal file
View file

@ -0,0 +1,121 @@
package internals
import (
"context"
"crypto/tls"
"errors"
"fmt"
"log"
"net/http"
"github.com/gregjones/httpcache"
"github.com/gregjones/httpcache/leveldbcache"
loghttp "github.com/motemen/go-loghttp"
"github.com/syndtr/goleveldb/leveldb"
"golang.org/x/oauth2"
)
var (
googleOauthConfig = &oauth2.Config{
RedirectURL: "http://localhost:3000/callback",
ClientID: "CLIENTKEY",
ClientSecret: "SECRETKEY",
Scopes: []string{
"esi-skills.read_skillqueue.v1",
"esi-skills.read_skills.v1",
"esi-planets.manage_planets.v1",
"esi-wallet.read_character_wallet.v1",
"esi-location.read_location.v1",
"esi-clones.read_clones.v1",
"esi-universe.read_structures.v1",
},
Endpoint: oauth2.Endpoint{
AuthURL: "https://login.eveonline.com/oauth/authorize/",
TokenURL: "https://login.eveonline.com/oauth/token/",
},
}
// Some random string, random for each request
oauthStateString = "random"
)
// GetTokenURL -
func GetTokenURL() string {
url := googleOauthConfig.AuthCodeURL(oauthStateString, oauth2.AccessTypeOffline)
// https://eveonline-third-party-documentation.readthedocs.io/en/latest/sso/authentication.html
// response_type: Must be set to “code”.
url = url + "&response_type=code"
return url
}
// GetTemporaryClient -
func GetTemporaryClient(r *http.Request) (*http.Client, *oauth2.Token, error) {
state := r.FormValue("state")
if state != oauthStateString {
errorTxt := fmt.Sprintf("invalid oauth state, expected '%s', got '%s'\n", oauthStateString, state)
return nil, nil, errors.New(errorTxt)
}
code := r.FormValue("code")
token, err := googleOauthConfig.Exchange(oauth2.NoContext, code)
if err != nil {
errorTxt := fmt.Sprintf("Code exchange failed with '%s'\n", err)
return nil, nil, errors.New(errorTxt)
}
client := googleOauthConfig.Client(oauth2.NoContext, token)
return client, token, nil
}
// GetDefaultClient - Returns the default client based on the given configuration
func GetDefaultClient(cacheDB *leveldb.DB, config *HTTPConfiguration) *http.Client {
ldb := leveldbcache.NewWithDB(cacheDB)
cachingTransport := httpcache.NewTransport(ldb)
if config.InsecureCalls {
insecureTransport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
}
cachingTransport.Transport = insecureTransport
}
if config.LogCalls {
var CustomLogResponse = func(resp *http.Response) {
log.Printf("<--- HTTP %d.%d %d %s (expires on %s) %s",
resp.ProtoMajor, resp.ProtoMinor,
resp.StatusCode,
resp.Request.URL,
resp.Header.Get("expires"),
resp.Header.Get("content-type"),
)
}
loggingTransport := &loghttp.Transport{
LogResponse: CustomLogResponse,
}
loggingTransport.Transport = cachingTransport.Transport
cachingTransport.Transport = loggingTransport
}
cachingClient := &http.Client{Transport: cachingTransport}
ctx := context.WithValue(context.TODO(), oauth2.HTTPClient, cachingClient)
googleOauthConfig.ClientID = config.ClientID
googleOauthConfig.ClientSecret = config.ClientSecret
client := googleOauthConfig.Client(ctx, config.ConnectionToken)
return client
}
// HTTPConfiguration - Configuration for the HTTP swagger client.
type HTTPConfiguration struct {
ClientID string
ClientSecret string
LogCalls bool
InsecureCalls bool
ConnectionToken *oauth2.Token
}

252
main.go
View file

@ -1,7 +1,6 @@
package main
import (
"crypto/tls"
"encoding/json"
"sort"
"time"
@ -17,16 +16,15 @@ import (
"fmt"
"log"
"os"
"strconv"
"github.com/BurntSushi/toml"
"github.com/go-openapi/strfmt"
"github.com/gregjones/httpcache"
"github.com/gregjones/httpcache/leveldbcache"
"github.com/leekchan/accounting"
"github.com/logrusorgru/aurora"
"github.com/syndtr/goleveldb/leveldb"
InternalUtils "./internals/"
ESI "./client"
ESIClones "./client/clones"
@ -37,8 +35,6 @@ import (
ESIWallet "./client/wallet"
httptransport "github.com/go-openapi/runtime/client"
loghttp "github.com/motemen/go-loghttp"
)
// Character - Structure to save the verification data.
@ -54,27 +50,6 @@ type configurationFile struct {
}
var (
googleOauthConfig = &oauth2.Config{
RedirectURL: "http://localhost:3000/callback",
ClientID: "CLIENTKEY",
ClientSecret: "SECRETKEY",
Scopes: []string{
"esi-skills.read_skillqueue.v1",
"esi-skills.read_skills.v1",
"esi-planets.manage_planets.v1",
"esi-wallet.read_character_wallet.v1",
"esi-location.read_location.v1",
"esi-clones.read_clones.v1",
"esi-universe.read_structures.v1",
},
Endpoint: oauth2.Endpoint{
AuthURL: "https://login.eveonline.com/oauth/authorize/",
TokenURL: "https://login.eveonline.com/oauth/token/",
},
}
// Some random string, random for each request
oauthStateString = "random"
defaultDateFormat = "_2 Jan 2006, 15:04"
cfgFilePath = flag.String("config", "configuration.toml", "Path to the configuration file.")
@ -85,7 +60,7 @@ var (
var ctx = context.Background()
var messages = make(chan *oauth2.Token)
var cacheDB *leveldb.DB
var cache InternalUtils.CacheConn
func main() {
flag.Parse()
@ -100,13 +75,15 @@ func main() {
log.SetOutput(f)
var cacheDBerr error
cacheDB, cacheDBerr = leveldb.OpenFile(*cacheDBPath, nil)
cacheDB, cacheDBerr := leveldb.OpenFile(*cacheDBPath, nil)
if cacheDBerr != nil {
fmt.Println("Unable to initialize the LevelDB cache.")
log.Fatal(cacheDBerr)
}
cErr := readConfigurationFile()
cache.SetCache(cacheDB)
httpConfiguration, cErr := readConfigurationFile()
if cErr != nil {
fmt.Println("Missing configuration file.")
log.Fatal(cErr)
@ -118,45 +95,17 @@ func main() {
cToken = getNewAuthorizationToken()
}
ldb := leveldbcache.NewWithDB(cacheDB)
cachingTransport := httpcache.NewTransport(ldb)
httpConfiguration.ConnectionToken = cToken
if *insecureFlag {
insecureTransport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
}
cachingTransport.Transport = insecureTransport
}
if *logCalls {
var CustomLogResponse = func(resp *http.Response) {
log.Printf("<--- HTTP %d.%d %d %s (expires on %s) %s",
resp.ProtoMajor, resp.ProtoMinor,
resp.StatusCode,
resp.Request.URL,
resp.Header.Get("expires"),
resp.Header.Get("content-type"),
)
}
loggingTransport := &loghttp.Transport{
LogResponse: CustomLogResponse,
}
loggingTransport.Transport = cachingTransport.Transport
cachingTransport.Transport = loggingTransport
}
cachingClient := &http.Client{Transport: cachingTransport}
ctx := context.WithValue(context.TODO(), oauth2.HTTPClient, cachingClient)
client := googleOauthConfig.Client(ctx, cToken)
client := InternalUtils.GetDefaultClient(cacheDB, httpConfiguration)
m, err := getCharacterInfo(client)
if err != nil {
log.Println(err)
cToken = getNewAuthorizationToken()
client = googleOauthConfig.Client(ctx, cToken)
httpConfiguration.ConnectionToken = cToken
client = InternalUtils.GetDefaultClient(cacheDB, httpConfiguration)
m, err = getCharacterInfo(client)
if err != nil {
log.Fatal(err)
@ -198,22 +147,26 @@ func main() {
}
}
func readConfigurationFile() error {
func readConfigurationFile() (*InternalUtils.HTTPConfiguration, error) {
var config configurationFile
if _, err := toml.DecodeFile(*cfgFilePath, &config); err != nil {
log.Println(err)
return err
return nil, err
}
if config.ClientID == "" || config.ClientSecret == "" {
log.Println("Missing ClientID or ClientSecret configuration option in configuration file" + *cfgFilePath + ".")
return errors.New("Missing ClientID or ClientSecret in configuration file " + *cfgFilePath + ".")
err := errors.New("Missing ClientID or ClientSecret in configuration file " + *cfgFilePath + ".")
return nil, err
}
googleOauthConfig.ClientID = config.ClientID
googleOauthConfig.ClientSecret = config.ClientSecret
var httpConfig InternalUtils.HTTPConfiguration
httpConfig.ClientID = config.ClientID
httpConfig.ClientSecret = config.ClientSecret
return nil
return &httpConfig, nil
}
func getCharacterInfo(client *http.Client) (*Character, error) {
@ -426,8 +379,8 @@ type structureInfo struct {
}
func getStructureStationInfo(swaggerclient *ESI.App, typeID string, ID int64) (*structureInfo, error) {
locationName := getCachedDataSub64(ID, "name")
locationSystem := getCachedDataIntSub64(ID, "system")
locationName := cache.GetCachedDataSub64(ID, "name")
locationSystem := cache.GetCachedDataIntSub64(ID, "system")
if locationName != "" && locationSystem > 0 {
var sInfo structureInfo
@ -458,8 +411,8 @@ func getStructureStationInfo(swaggerclient *ESI.App, typeID string, ID int64) (*
sInfo.SystemID = *staInfo.SystemID
sInfo.System = getSolarSystemInformation(swaggerclient, sInfo.SystemID)
putCacheDataSub64(sInfo.ID, "name", sInfo.Name)
putCacheDataIntSub64(sInfo.ID, "system", sInfo.SystemID)
cache.PutCacheDataSub64(sInfo.ID, "name", sInfo.Name)
cache.PutCacheDataIntSub64(sInfo.ID, "system", sInfo.SystemID)
return &sInfo, nil
}
@ -483,8 +436,8 @@ func getStructureStationInfo(swaggerclient *ESI.App, typeID string, ID int64) (*
sInfo.SystemID = *strInfo.SolarSystemID
sInfo.System = getSolarSystemInformation(swaggerclient, sInfo.SystemID)
putCacheDataSub64(sInfo.ID, "name", sInfo.Name)
putCacheDataIntSub64(sInfo.ID, "system", sInfo.SystemID)
cache.PutCacheDataSub64(sInfo.ID, "name", sInfo.Name)
cache.PutCacheDataIntSub64(sInfo.ID, "system", sInfo.SystemID)
return &sInfo, nil
}
@ -614,7 +567,7 @@ func getUniverseNames(swaggerclient *ESI.App, itemIds *[]int32) map[int32]string
itemMissingNames := make(map[int32]string)
for _, itemID := range *itemIds {
itemName := getCachedDataSub(itemID, "name")
itemName := cache.GetCachedDataSub(itemID, "name")
if itemName != "" {
itemNames[itemID] = itemName
} else {
@ -640,7 +593,7 @@ func getUniverseNames(swaggerclient *ESI.App, itemIds *[]int32) map[int32]string
itemName := searchResult.Name
itemID := searchResult.ID
putCacheDataSub(*itemID, "name", *itemName)
cache.PutCacheDataSub(*itemID, "name", *itemName)
itemNames[*itemID] = *itemName
}
@ -657,8 +610,8 @@ type planetInformation struct {
func getPlanetInformation(swaggerclient *ESI.App, planetID int32) planetInformation {
planetName := getCachedDataSub(planetID, "name")
planetType := getCachedDataIntSub(planetID, "type")
planetName := cache.GetCachedDataSub(planetID, "name")
planetType := cache.GetCachedDataIntSub(planetID, "type")
if planetName == "" {
pcallParams := ESIUniverse.NewGetUniversePlanetsPlanetIDParams()
@ -672,8 +625,8 @@ func getPlanetInformation(swaggerclient *ESI.App, planetID int32) planetInformat
planetInfo.PlanetName = *planetESIInfo.Name
planetInfo.PlanetTypeID = *planetESIInfo.TypeID
putCacheDataSub(planetID, "name", planetInfo.PlanetName)
putCacheDataIntSub(planetID, "type", planetInfo.PlanetTypeID)
cache.PutCacheDataSub(planetID, "name", planetInfo.PlanetName)
cache.PutCacheDataIntSub(planetID, "type", planetInfo.PlanetTypeID)
return planetInfo
}
@ -695,11 +648,11 @@ type solarSystemInfo struct {
func getSolarSystemInformation(swaggerclient *ESI.App, solarSystemID int32) solarSystemInfo {
systemName := getCachedDataSub(solarSystemID, "name")
systemConstellationID := getCachedDataIntSub(solarSystemID, "constellation")
constellationName := getCachedDataSub(systemConstellationID, "name")
constellationRegionID := getCachedDataIntSub(systemConstellationID, "region")
regionName := getCachedDataSub(constellationRegionID, "name")
systemName := cache.GetCachedDataSub(solarSystemID, "name")
systemConstellationID := cache.GetCachedDataIntSub(solarSystemID, "constellation")
constellationName := cache.GetCachedDataSub(systemConstellationID, "name")
constellationRegionID := cache.GetCachedDataIntSub(systemConstellationID, "region")
regionName := cache.GetCachedDataSub(constellationRegionID, "name")
if systemName == "" || constellationName == "" || regionName == "" {
scallParams := ESIUniverse.NewGetUniverseSystemsSystemIDParams()
@ -729,13 +682,13 @@ func getSolarSystemInformation(swaggerclient *ESI.App, solarSystemID int32) sola
ssInfo.RegionName = *regionESIInfo.Name
putCacheDataSub(solarSystemID, "name", ssInfo.SolarSystemName)
putCacheDataIntSub(solarSystemID, "constellation", ssInfo.ConstellationID)
cache.PutCacheDataSub(solarSystemID, "name", ssInfo.SolarSystemName)
cache.PutCacheDataIntSub(solarSystemID, "constellation", ssInfo.ConstellationID)
putCacheDataSub(ssInfo.ConstellationID, "name", ssInfo.ConstellationName)
putCacheDataIntSub(ssInfo.ConstellationID, "region", ssInfo.RegionID)
cache.PutCacheDataSub(ssInfo.ConstellationID, "name", ssInfo.ConstellationName)
cache.PutCacheDataIntSub(ssInfo.ConstellationID, "region", ssInfo.RegionID)
putCacheDataSub(ssInfo.RegionID, "name", ssInfo.RegionName)
cache.PutCacheDataSub(ssInfo.RegionID, "name", ssInfo.RegionName)
return ssInfo
}
@ -758,8 +711,8 @@ type SchematicInfo struct {
func getSchematicsInformation(swaggerclient *ESI.App, schematicID int32) (*SchematicInfo, error) {
schematicName := getCachedDataSub(schematicID, "name")
schematicCycle := getCachedDataIntSub(schematicID, "cycle")
schematicName := cache.GetCachedDataSub(schematicID, "name")
schematicCycle := cache.GetCachedDataIntSub(schematicID, "cycle")
if schematicName == "" {
scallParams := ESIPlanetaryInteraction.NewGetUniverseSchematicsSchematicIDParams()
@ -776,8 +729,8 @@ func getSchematicsInformation(swaggerclient *ESI.App, schematicID int32) (*Schem
schematicInfo.CycleTime = *schematicsESIInfo.CycleTime
schematicInfo.SchematicName = *schematicsESIInfo.SchematicName
putCacheDataSub(schematicID, "name", schematicInfo.SchematicName)
putCacheDataIntSub(schematicID, "cycle", schematicInfo.CycleTime)
cache.PutCacheDataSub(schematicID, "name", schematicInfo.SchematicName)
cache.PutCacheDataIntSub(schematicID, "cycle", schematicInfo.CycleTime)
return &schematicInfo, nil
}
@ -801,123 +754,38 @@ func getNewAuthorizationToken() *oauth2.Token {
}
func handleLogin(w http.ResponseWriter, r *http.Request) {
url := googleOauthConfig.AuthCodeURL(oauthStateString, oauth2.AccessTypeOffline)
// https://eveonline-third-party-documentation.readthedocs.io/en/latest/sso/authentication.html
// response_type: Must be set to “code”.
url = url + "&response_type=code"
url := InternalUtils.GetTokenURL()
http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}
func handleAuthenticationCallback(w http.ResponseWriter, r *http.Request) {
state := r.FormValue("state")
if state != oauthStateString {
log.Printf("invalid oauth state, expected '%s', got '%s'\n", oauthStateString, state)
client, token, clientErr := InternalUtils.GetTemporaryClient(r)
if clientErr != nil {
log.Printf("Code exchange failed with '%s'\n", clientErr)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
// No token to pass, we will get one on the second pass
return
}
code := r.FormValue("code")
token, err := googleOauthConfig.Exchange(oauth2.NoContext, code)
if err != nil {
log.Printf("Code exchange failed with '%s'\n", err)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
// No token to pass, we will get one on the second pass
return
}
client := googleOauthConfig.Client(oauth2.NoContext, token)
req, _ := http.NewRequest("GET", "https://login.eveonline.com/oauth/verify", nil)
response, _ := client.Do(req)
defer response.Body.Close()
contents, _ := ioutil.ReadAll(response.Body)
var m Character
errJSON := json.Unmarshal(contents, &m)
if errJSON != nil {
fmt.Printf("JSON read error with '%s'\n", errJSON)
charInfo, charErr := getCharacterInfo(client)
if charErr != nil {
fmt.Printf("Character read error with '%s'\n", charErr)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
return
}
fmt.Fprintf(w, "Got token for character %s.\n", m.CharacterName)
fmt.Fprintf(w, "Got token for character %s.\n", charInfo.CharacterName)
fmt.Fprintf(w, "You can now close this navigator tab.\n")
log.Printf("Refresh token is %s\n", token.RefreshToken)
putCacheData("accessToken", token.AccessToken)
putCacheData("refreshToken", token.RefreshToken)
cache.PutCacheData("accessToken", token.AccessToken)
cache.PutCacheData("refreshToken", token.RefreshToken)
messages <- token
}
func getCachedDataIntSub(key int32, subkey string) int32 {
return getCachedDataInt(fmt.Sprintf("%d.%s", key, subkey))
}
func getCachedDataSub(key int32, subkey string) string {
return getCachedData(fmt.Sprintf("%d.%s", key, subkey))
}
func getCachedDataIntSub64(key int64, subkey string) int32 {
return getCachedDataInt(fmt.Sprintf("%d.%s", key, subkey))
}
func getCachedDataSub64(key int64, subkey string) string {
return getCachedData(fmt.Sprintf("%d.%s", key, subkey))
}
func getCachedDataInt(key string) int32 {
strVal := getCachedData(key)
intVal, convErr := strconv.ParseInt(strVal, 10, 32)
if convErr != nil {
return -1
}
return int32(intVal)
}
func getCachedData(key string) string {
response, err := cacheDB.Get([]byte(key), nil)
if err != nil {
return ""
}
return string(response[:])
}
func putCacheDataIntSub(key int32, subkey string, value int32) {
putCacheDataSub(key, subkey, fmt.Sprintf("%d", value))
}
func putCacheDataSub(key int32, subkey string, value string) {
putCacheData(fmt.Sprintf("%d.%s", key, subkey), value)
}
func putCacheDataIntSub64(key int64, subkey string, value int32) {
putCacheDataSub64(key, subkey, fmt.Sprintf("%d", value))
}
func putCacheDataSub64(key int64, subkey string, value string) {
putCacheData(fmt.Sprintf("%d.%s", key, subkey), value)
}
func putCacheData(key string, value string) {
err := cacheDB.Put([]byte(key), []byte(value), nil)
if err != nil {
log.Fatal(err)
}
}
func getDatabaseToken() *oauth2.Token {
refreshToken := getCachedData("refreshToken")
//accessToken := getCachedData("accessToken")
refreshToken := cache.GetCachedData("refreshToken")
//accessToken := cache.GetCachedData("accessToken")
token := new(oauth2.Token)
token.RefreshToken = refreshToken