generator.go

package openapi

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"

	"github.com/clbanning/mxj/v2"
	"github.com/getkin/kin-openapi/openapi3"
	"github.com/pkg/errors"
	"github.com/projectdiscovery/gologger"
	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
	httpTypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
	"github.com/projectdiscovery/nuclei/v3/pkg/types"
	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
	"github.com/projectdiscovery/utils/errkit"
	"github.com/projectdiscovery/utils/generic"
	mapsutil "github.com/projectdiscovery/utils/maps"
	"github.com/valyala/fasttemplate"
)

const (
	globalAuth                 = "globalAuth"
	DEFAULT_HTTP_SCHEME_HEADER = "Authorization"
)

// GenerateRequestsFromSchema generates http requests from an OpenAPI 3.0 document object
func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOptions, callback formats.ParseReqRespCallback) error {
	if len(schema.Servers) == 0 {
		return errors.New("no servers found in openapi schema")
	}

	// new set of globalParams obtained from security schemes
	globalParams := openapi3.NewParameters()

	if len(schema.Security) > 0 {
		params, err := GetGlobalParamsForSecurityRequirement(schema, &schema.Security)
		if err != nil {
			return err
		}
		globalParams = append(globalParams, params...)
	}

	// validate global param requirements
	for _, param := range globalParams {
		if val, ok := opts.Variables[param.Value.Name]; ok {
			param.Value.Example = val
		} else {
			// if missing check for validation
			if opts.SkipFormatValidation {
				gologger.Verbose().Msgf("openapi: skipping all requests due to missing global auth parameter: %s\n", param.Value.Name)
				return nil
			} else {
				// fatal error
				gologger.Fatal().Msgf("openapi: missing global auth parameter: %s\n", param.Value.Name)
			}
		}
	}

	missingVarMap := make(map[string]struct{})
	optionalVarMap := make(map[string]struct{})
	missingParamValueCallback := func(param *openapi3.Parameter, opts *generateReqOptions) {
		if !param.Required {
			optionalVarMap[param.Name] = struct{}{}
			return
		}
		missingVarMap[param.Name] = struct{}{}
	}

	for _, serverURL := range schema.Servers {
		pathURL := serverURL.URL
		// Split the server URL into baseURL and serverPath
		u, err := url.Parse(pathURL)
		if err != nil {
			return errors.Wrap(err, "could not parse server url")
		}
		baseURL := fmt.Sprintf("%s://%s", u.Scheme, u.Host)
		serverPath := u.Path

		for path, v := range schema.Paths.Map() {
			// a path item can have parameters
			ops := v.Operations()
			requestPath := path
			if serverPath != "" {
				requestPath = serverPath + path
			}
			for method, ov := range ops {
				if err := generateRequestsFromOp(&generateReqOptions{
					requiredOnly:              opts.RequiredOnly,
					method:                    method,
					pathURL:                   baseURL,
					requestPath:               requestPath,
					op:                        ov,
					schema:                    schema,
					globalParams:              globalParams,
					reqParams:                 v.Parameters,
					opts:                      opts,
					callback:                  callback,
					missingParamValueCallback: missingParamValueCallback,
				}); err != nil {
					gologger.Warning().Msgf("Could not generate requests from op: %s\n", err)
				}
			}
		}
	}

	if len(missingVarMap) > 0 && !opts.SkipFormatValidation {
		gologger.Error().Msgf("openapi: Found %d missing parameters, use -skip-format-validation flag to skip requests or update missing parameters generated in %s file,you can also specify these vars using -var flag in (key=value) format\n", len(missingVarMap), formats.DefaultVarDumpFileName)
		gologger.Verbose().Msgf("openapi: missing params: %+v", mapsutil.GetSortedKeys(missingVarMap))
		if config.CurrentAppMode == config.AppModeCLI {
			// generate var dump file
			vars := &formats.OpenAPIParamsCfgFile{}
			for k := range missingVarMap {
				vars.Var = append(vars.Var, k+"=")
			}
			vars.OptionalVars = mapsutil.GetSortedKeys(optionalVarMap)
			if err := formats.WriteOpenAPIVarDumpFile(vars); err != nil {
				gologger.Error().Msgf("openapi: could not write params file: %s\n", err)
			}
			// exit with status code 1
			os.Exit(1)
		}
	}

	return nil
}

type generateReqOptions struct {
	// requiredOnly specifies whether to generate only required fields
	requiredOnly bool
	// method is the http method to use
	method string
	// pathURL is the base url to use
	pathURL string
	// requestPath is the path to use
	requestPath string
	// schema is the openapi schema to use
	schema *openapi3.T
	// op is the operation to use
	op *openapi3.Operation
	// post request generation callback
	callback formats.ParseReqRespCallback

	// global parameters
	globalParams openapi3.Parameters
	// requestparams map
	reqParams openapi3.Parameters
	// global var map
	opts formats.InputFormatOptions
	// missingVar Callback
	missingParamValueCallback func(param *openapi3.Parameter, opts *generateReqOptions)
}

// generateRequestsFromOp generates requests from an operation and some other data
// about an OpenAPI Schema Path and Method object.
//
// It also accepts an optional requiredOnly flag which if specified, only returns the fields
// of the structure that are required. If false, all fields are returned.
func generateRequestsFromOp(opts *generateReqOptions) error {
	req, err := http.NewRequest(opts.method, opts.pathURL+opts.requestPath, nil)
	if err != nil {
		return errors.Wrap(err, "could not make request")
	}

	reqParams := opts.reqParams
	if reqParams == nil {
		reqParams = openapi3.NewParameters()
	}
	// add existing req params
	reqParams = append(reqParams, opts.op.Parameters...)
	// check for endpoint specific auth
	if opts.op.Security != nil {
		params, err := GetGlobalParamsForSecurityRequirement(opts.schema, opts.op.Security)
		if err != nil {
			return err
		}
		reqParams = append(reqParams, params...)
	} else {
		reqParams = append(reqParams, opts.globalParams...)
	}

	query := url.Values{}
	for _, parameter := range reqParams {
		value := parameter.Value

		if value.Schema == nil || value.Schema.Value == nil {
			continue
		}

		// paramValue or default value to use
		var paramValue interface{}

		// accept override from global variables
		if val, ok := opts.opts.Variables[value.Name]; ok {
			paramValue = val
		} else if value.Schema.Value.Default != nil {
			paramValue = value.Schema.Value.Default
		} else if value.Schema.Value.Example != nil {
			paramValue = value.Schema.Value.Example
		} else if len(value.Schema.Value.Enum) > 0 {
			paramValue = value.Schema.Value.Enum[0]
		} else {
			if !opts.opts.SkipFormatValidation {
				if opts.missingParamValueCallback != nil {
					opts.missingParamValueCallback(value, opts)
				}
				// skip request if param in path else skip this param only
				if value.Required {
					// gologger.Verbose().Msgf("skipping request [%s] %s due to missing value (%v)\n", opts.method, opts.requestPath, value.Name)
					return nil
				} else {
					// if it is in path then remove it from path
					opts.requestPath = strings.ReplaceAll(opts.requestPath, fmt.Sprintf("{%s}", value.Name), "")
					if !opts.opts.RequiredOnly {
						gologger.Verbose().Msgf("openapi: skipping optional param (%s) in (%v) in request [%s] %s due to missing value (%v)\n", value.Name, value.In, opts.method, opts.requestPath, value.Name)
					}
					continue
				}
			}
			exampleX, err := generateExampleFromSchema(value.Schema.Value)
			if err != nil {
				// when failed to generate example
				// skip request if param in path else skip this param only
				if value.Required {
					gologger.Verbose().Msgf("openapi: skipping request [%s] %s due to missing value (%v)\n", opts.method, opts.requestPath, value.Name)
					return nil
				} else {
					// if it is in path then remove it from path
					opts.requestPath = strings.ReplaceAll(opts.requestPath, fmt.Sprintf("{%s}", value.Name), "")
					if !opts.opts.RequiredOnly {
						gologger.Verbose().Msgf("openapi: skipping optional param (%s) in (%v) in request [%s] %s due to missing value (%v)\n", value.Name, value.In, opts.method, opts.requestPath, value.Name)
					}
					continue
				}
			}
			paramValue = exampleX
		}
		if opts.requiredOnly && !value.Required {
			// remove them from path if any
			opts.requestPath = strings.ReplaceAll(opts.requestPath, fmt.Sprintf("{%s}", value.Name), "")
			continue // Skip this parameter if it is not required and we want only required ones
		}

		switch value.In {
		case "query":
			query.Set(value.Name, types.ToString(paramValue))
		case "header":
			req.Header.Set(value.Name, types.ToString(paramValue))
		case "path":
			opts.requestPath = fasttemplate.ExecuteStringStd(opts.requestPath, "{", "}", map[string]interface{}{
				value.Name: types.ToString(paramValue),
			})
		case "cookie":
			req.AddCookie(&http.Cookie{Name: value.Name, Value: types.ToString(paramValue)})
		}
	}
	req.URL.RawQuery = query.Encode()
	req.URL.Path = opts.requestPath

	if opts.op.RequestBody != nil {
		for content, value := range opts.op.RequestBody.Value.Content {
			cloned := req.Clone(req.Context())

			var val interface{}

			if value.Schema == nil || value.Schema.Value == nil {
				val = generateEmptySchemaValue(content)
			} else {
				var err error

				val, err = generateExampleFromSchema(value.Schema.Value)
				if err != nil {
					continue
				}
			}

			// var body string
			switch content {
			case "application/json":
				if marshalled, err := json.Marshal(val); err == nil {
					// body = string(marshalled)
					cloned.Body = io.NopCloser(bytes.NewReader(marshalled))
					cloned.ContentLength = int64(len(marshalled))
					cloned.Header.Set("Content-Type", "application/json")
				}
			case "application/xml":
				values := mxj.Map(val.(map[string]interface{}))

				if marshalled, err := values.Xml(); err == nil {
					// body = string(marshalled)
					cloned.Body = io.NopCloser(bytes.NewReader(marshalled))
					cloned.ContentLength = int64(len(marshalled))
					cloned.Header.Set("Content-Type", "application/xml")
				} else {
					gologger.Warning().Msgf("openapi: could not encode xml")
				}
			case "application/x-www-form-urlencoded":
				if values, ok := val.(map[string]interface{}); ok {
					cloned.Form = url.Values{}
					for k, v := range values {
						cloned.Form.Set(k, types.ToString(v))
					}
					encoded := cloned.Form.Encode()
					cloned.ContentLength = int64(len(encoded))
					// body = encoded
					cloned.Body = io.NopCloser(strings.NewReader(encoded))
					cloned.Header.Set("Content-Type", "application/x-www-form-urlencoded")
				}
			case "multipart/form-data":
				if values, ok := val.(map[string]interface{}); ok {
					buffer := &bytes.Buffer{}
					multipartWriter := multipart.NewWriter(buffer)
					for k, v := range values {
						// This is a file if format is binary, otherwise field
						if property, ok := value.Schema.Value.Properties[k]; ok && property.Value.Format == "binary" {
							if writer, err := multipartWriter.CreateFormFile(k, k); err == nil {
								_, _ = writer.Write([]byte(types.ToString(v)))
							}
						} else {
							_ = multipartWriter.WriteField(k, types.ToString(v))
						}
					}
					_ = multipartWriter.Close()
					// body = buffer.String()
					cloned.Body = io.NopCloser(buffer)
					cloned.ContentLength = int64(len(buffer.Bytes()))
					cloned.Header.Set("Content-Type", multipartWriter.FormDataContentType())
				}
			case "text/plain":
				str := types.ToString(val)
				// body = str
				cloned.Body = io.NopCloser(strings.NewReader(str))
				cloned.ContentLength = int64(len(str))
				cloned.Header.Set("Content-Type", "text/plain")
			case "application/octet-stream":
				str := types.ToString(val)
				if str == "" {
					// use two strings
					str = "string1\nstring2"
				}
				if value.Schema != nil && generic.EqualsAny(value.Schema.Value.Format, "bindary", "byte") {
					cloned.Body = io.NopCloser(bytes.NewReader([]byte(str)))
					cloned.ContentLength = int64(len(str))
					cloned.Header.Set("Content-Type", "application/octet-stream")
				} else {
					// use string placeholder
					cloned.Body = io.NopCloser(strings.NewReader(str))
					cloned.ContentLength = int64(len(str))
					cloned.Header.Set("Content-Type", "text/plain")
				}
			default:
				gologger.Verbose().Msgf("openapi: no correct content type found for body: %s\n", content)
				// LOG:	return errors.New("no correct content type found for body")
				continue
			}

			dumped, err := httputil.DumpRequestOut(cloned, true)
			if err != nil {
				return errors.Wrap(err, "could not dump request")
			}

			rr, err := httpTypes.ParseRawRequestWithURL(string(dumped), cloned.URL.String())
			if err != nil {
				return errors.Wrap(err, "could not parse raw request")
			}
			opts.callback(rr)
			continue
		}
	}
	if opts.op.RequestBody != nil {
		return nil
	}

	dumped, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		return errors.Wrap(err, "could not dump request")
	}

	rr, err := httpTypes.ParseRawRequestWithURL(string(dumped), req.URL.String())
	if err != nil {
		return errors.Wrap(err, "could not parse raw request")
	}
	opts.callback(rr)
	return nil
}

// GetGlobalParamsForSecurityRequirement returns the global parameters for a security requirement
func GetGlobalParamsForSecurityRequirement(schema *openapi3.T, requirement *openapi3.SecurityRequirements) ([]*openapi3.ParameterRef, error) {
	globalParams := openapi3.NewParameters()
	if len(schema.Components.SecuritySchemes) == 0 {
		return nil, errkit.Newf("security requirements (%+v) without any security schemes found in openapi file", schema.Security)
	}
	found := false
	// this api is protected for each security scheme pull its corresponding scheme
schemaLabel:
	for _, security := range *requirement {
		for name := range security {
			if scheme, ok := schema.Components.SecuritySchemes[name]; ok {
				found = true
				param, err := GenerateParameterFromSecurityScheme(scheme)
				if err != nil {
					return nil, err

				}
				globalParams = append(globalParams, &openapi3.ParameterRef{Value: param})
				continue schemaLabel
			}
		}
		if !found && len(security) > 1 {
			// if this is case then both security schemes are required
			return nil, errkit.Newf("security requirement (%+v) not found in openapi file", security)
		}
	}
	if !found {
		return nil, errkit.Newf("security requirement (%+v) not found in openapi file", requirement)
	}

	return globalParams, nil
}

// GenerateParameterFromSecurityScheme generates an example from a schema object
func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*openapi3.Parameter, error) {
	if !generic.EqualsAny(scheme.Value.Type, "http", "apiKey") {
		return nil, errkit.Newf("unsupported security scheme type (%s) found in openapi file", scheme.Value.Type)
	}
	if scheme.Value.Type == "http" {
		// check scheme
		if !generic.EqualsAny(scheme.Value.Scheme, "basic", "bearer") {
			return nil, errkit.Newf("unsupported security scheme (%s) found in openapi file", scheme.Value.Scheme)
		}
		// HTTP authentication schemes basic or bearer use the Authorization header
		headerName := scheme.Value.Name
		if headerName == "" {
			headerName = DEFAULT_HTTP_SCHEME_HEADER
		}
		// create parameters using the scheme
		switch scheme.Value.Scheme {
		case "basic":
			h := openapi3.NewHeaderParameter(headerName)
			h.Required = true
			h.Description = globalAuth // differentiator for normal variables and global auth
			return h, nil
		case "bearer":
			h := openapi3.NewHeaderParameter(headerName)
			h.Required = true
			h.Description = globalAuth // differentiator for normal variables and global auth
			return h, nil
		}

	}
	if scheme.Value.Type == "apiKey" {
		// validate name and in
		if scheme.Value.Name == "" {
			return nil, errkit.Newf("security scheme (%s) name is empty", scheme.Value.Type)
		}
		if !generic.EqualsAny(scheme.Value.In, "query", "header", "cookie") {
			return nil, errkit.Newf("unsupported security scheme (%s) in (%s) found in openapi file", scheme.Value.Type, scheme.Value.In)
		}
		// create parameters using the scheme
		switch scheme.Value.In {
		case "query":
			q := openapi3.NewQueryParameter(scheme.Value.Name)
			q.Required = true
			q.Description = globalAuth // differentiator for normal variables and global auth
			return q, nil
		case "header":
			h := openapi3.NewHeaderParameter(scheme.Value.Name)
			h.Required = true
			h.Description = globalAuth // differentiator for normal variables and global auth
			return h, nil
		case "cookie":
			c := openapi3.NewCookieParameter(scheme.Value.Name)
			c.Required = true
			c.Description = globalAuth // differentiator for normal variables and global auth
			return c, nil
		}
	}
	return nil, errkit.Newf("unsupported security scheme type (%s) found in openapi file", scheme.Value.Type)
}