package discord
import (
"bufio"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
"time"
"github.com/V4NSH4J/discord-mass-dm-GO/instance"
"github.com/V4NSH4J/discord-mass-dm-GO/utilities"
)
func LaunchScraperMenu() {
cfg, _, err := instance.GetEverything()
if err != nil {
utilities.LogErr("Error while getting neccessary information %v", err)
utilities.ExitSafely()
}
utilities.PrintMenu([]string{"Online Scraper (Opcode 14)", "Scrape from Reactions (REST API)", "Offline Scraper (Opcode 8)"})
options := utilities.UserInputInteger("Select an option: ")
if options == 1 {
token := utilities.UserInput("Enter the token: ")
serverid := utilities.UserInput("Enter the server ID: ")
channelid := utilities.UserInput("Enter the channel ID: ")
var botsFile, avatarFile, nameFile, path, rolePath, scrapedFile, userDataFile string
if cfg.OtherSettings.Logs {
path = fmt.Sprintf(`logs/online_scraper/DMDGO-OS-%s-%s-%s-%s`, serverid, channelid, time.Now().Format(`2006-01-02 15-04-05`), utilities.RandStringBytes(5))
rolePath = fmt.Sprintf(`%s/roles`, path)
err := os.MkdirAll(path, 0755)
if err != nil && !os.IsExist(err) {
utilities.LogErr("Error creating logs directory: %s", err)
utilities.ExitSafely()
}
err = os.MkdirAll(rolePath, 0755)
if err != nil && !os.IsExist(err) {
utilities.LogErr("Error creating roles directory: %s", err)
utilities.ExitSafely()
}
botsFileX, err := os.Create(fmt.Sprintf(`%s/bots.txt`, path))
if err != nil {
utilities.LogErr("Error creating bots file: %s", err)
utilities.ExitSafely()
}
botsFileX.Close()
AvatarFileX, err := os.Create(fmt.Sprintf(`%s/avatars.txt`, path))
if err != nil {
utilities.LogErr("Error creating avatars file: %s", err)
utilities.ExitSafely()
}
AvatarFileX.Close()
NameFileX, err := os.Create(fmt.Sprintf(`%s/names.txt`, path))
if err != nil {
utilities.LogErr("Error creating names file: %s", err)
utilities.ExitSafely()
}
NameFileX.Close()
ScrapedFileX, err := os.Create(fmt.Sprintf(`%s/scraped.txt`, path))
if err != nil {
utilities.LogErr("Error creating scraped file: %s", err)
utilities.ExitSafely()
}
UserDataFileX, err := os.Create(fmt.Sprintf(`%s/user_data.txt`, path))
if err != nil {
utilities.LogErr("Error creating user data file: %s", err)
utilities.ExitSafely()
}
botsFile, avatarFile, nameFile, scrapedFile, userDataFile = botsFileX.Name(), AvatarFileX.Name(), NameFileX.Name(), ScrapedFileX.Name(), UserDataFileX.Name()
}
Is := instance.Instance{Token: token}
title := make(chan bool)
go func() {
Out:
for {
select {
case <-title:
break Out
default:
if Is.Ws != nil {
if Is.Ws.Conn != nil {
cmd := exec.Command("cmd", "/C", "title", fmt.Sprintf(`DMDGO [%v Scraped]`, len(Is.Ws.Members)))
_ = cmd.Run()
}
}
}
}
}()
t := 0
for {
if t >= 5 {
utilities.LogErr("Couldn't connect to websocket after retrying.")
break
}
err := Is.StartWS()
if err != nil {
utilities.LogFailed("Error while opening websocket: %v", err)
} else {
break
}
t++
}
utilities.LogErr("Websocket opened %v", Is.Token)
i := 0
for {
err := instance.Scrape(Is.Ws, serverid, channelid, i)
if err != nil {
utilities.LogErr("Error while scraping: %v", err)
}
utilities.LogSuccess("Token %v Scrape Count: %v", Is.Token, len(Is.Ws.Members))
if Is.Ws.Complete {
break
}
i++
time.Sleep(time.Duration(cfg.ScraperSettings.SleepSc) * time.Millisecond)
}
if Is.Ws != nil {
Is.Ws.Close()
}
utilities.LogSuccess("Scraping finished. Scraped %v members", len(Is.Ws.Members))
if cfg.OtherSettings.Logs {
for i := 0; i < len(Is.Ws.Members); i++ {
if Is.Ws.Members[i].User.Bot {
utilities.WriteLinesPath(botsFile, fmt.Sprintf("%v %v %v", Is.Ws.Members[i].User.ID, Is.Ws.Members[i].User.Username, Is.Ws.Members[i].User.Discriminator))
}
if Is.Ws.Members[i].User.Avatar != "" {
utilities.WriteLinesPath(avatarFile, fmt.Sprintf("%v:%v", Is.Ws.Members[i].User.ID, Is.Ws.Members[i].User.Avatar))
}
if Is.Ws.Members[i].User.Username != "" {
utilities.WriteLinesPath(nameFile, fmt.Sprintf("%v", Is.Ws.Members[i].User.Username))
}
for x := 0; x < len(Is.Ws.Members[i].Roles); x++ {
utilities.WriteRoleFile(Is.Ws.Members[i].User.ID, rolePath, Is.Ws.Members[i].Roles[x])
}
if Is.Ws.Members[i].User.Discriminator != "" && Is.Ws.Members[i].User.Username != "" {
utilities.WriteLinesPath(userDataFile, fmt.Sprintf("%v#%v", Is.Ws.Members[i].User.Username, Is.Ws.Members[i].User.Discriminator))
}
utilities.WriteLinesPath(scrapedFile, Is.Ws.Members[i].User.ID)
}
}
var memberids []string
for _, member := range Is.Ws.Members {
memberids = append(memberids, member.User.ID)
}
clean := utilities.RemoveDuplicateStr(memberids)
utilities.LogSuccess("Removed Duplicates. Scraped %v members", len(clean))
write := utilities.UserInput("Write to memberids.txt? (y/n)")
title <- true
if write == "y" {
for k := 0; k < len(clean); k++ {
err := utilities.WriteLines("memberids.txt", clean[k])
if err != nil {
utilities.LogErr("Error while writing to file: %v", err)
}
}
utilities.LogSuccess("Wrote %v members to memberids.txt", len(clean))
}
}
if options == 2 {
token := utilities.UserInput("Enter the token: ")
channelid := utilities.UserInput("Enter the channel ID: ")
messageid := utilities.UserInput("Enter the message ID: ")
utilities.PrintMenu([]string{"Get Emoji from Message", "Enter Emoji Manually"})
option := utilities.UserInputInteger("Select an option: ")
var send string
if option == 2 {
send = utilities.UserInput("Enter the emoji [Format emojiName or emojiName:emojiID for nitro emojis]: ")
} else {
msg, err := instance.GetRxn(channelid, messageid, token)
if err != nil {
utilities.LogErr("Error while getting message: %v", err)
}
var selection []string
for i := 0; i < len(msg.Reactions); i++ {
selection = append(selection, fmt.Sprintf("Emoji: %v | Count: %v", msg.Reactions[i].Emojis.Name, msg.Reactions[i].Count))
}
utilities.PrintMenu2(selection)
index := utilities.UserInputInteger("Select an option: ")
if msg.Reactions[index].Emojis.ID == "" {
send = msg.Reactions[index].Emojis.Name
} else if msg.Reactions[index].Emojis.ID != "" {
send = msg.Reactions[index].Emojis.Name + ":" + msg.Reactions[index].Emojis.ID
}
}
var allUIDS []string
var m string
title := make(chan bool)
go func() {
Out:
for {
select {
case <-title:
break Out
default:
cmd := exec.Command("cmd", "/C", "title", fmt.Sprintf(`DMDGO [%v Scraped]`, len(allUIDS)))
_ = cmd.Run()
}
}
}()
for {
if len(allUIDS) == 0 {
m = ""
} else {
m = allUIDS[len(allUIDS)-1]
}
rxn, err := instance.GetReactions(channelid, messageid, token, send, m)
if err != nil {
utilities.LogErr("Error while getting reactions: %v", err)
continue
}
if len(rxn) == 0 {
break
}
utilities.LogInfo("Scraped %v members", len(rxn))
allUIDS = append(allUIDS, rxn...)
}
utilities.LogInfo("Scraping finished. Scraped %v lines - Removing Duplicates", len(allUIDS))
clean := utilities.RemoveDuplicateStr(allUIDS)
path := fmt.Sprintf(`logs/reaction_scraper/DMDGO-RS-%s-%s-%s-%s`, channelid, messageid, time.Now().Format(`2006-01-02 15-04-05`), utilities.RandStringBytes(5))
err := os.MkdirAll(path, 0755)
if err != nil && !os.IsExist(err) {
utilities.LogErr("Error creating logs directory: %s", err)
utilities.ExitSafely()
}
scrapedFileX, err := os.Create(fmt.Sprintf(`%s/scraped.txt`, path))
if err != nil {
utilities.LogErr("Error creating scraped file: %s", err)
utilities.ExitSafely()
}
defer scrapedFileX.Close()
scrapedFile := scrapedFileX.Name()
for i := 0; i < len(clean); i++ {
utilities.WriteLinesPath(scrapedFile, clean[i])
}
write := utilities.UserInput("Write to memberids.txt? (y/n)")
if write == "y" {
for k := 0; k < len(clean); k++ {
err := utilities.WriteLines("memberids.txt", clean[k])
if err != nil {
utilities.LogErr("Error while writing to file: %v", err)
}
}
utilities.LogSuccess("Wrote %v members to memberids.txt", len(clean))
}
title <- true
utilities.LogSuccess("Finished")
}
if options == 3 {
cfg, instances, err := instance.GetEverything()
if err != nil {
utilities.LogErr("Error while getting instances: %v", err)
return
}
var scraped []string
var queriesCompleted []string
title := make(chan bool)
go func() {
Out:
for {
select {
case <-title:
break Out
default:
cmd := exec.Command("cmd", "/C", "title", fmt.Sprintf(`DMDGO [%v Scraped %v Queries Completed]`, len(scraped), len(queriesCompleted)))
_ = cmd.Run()
}
}
}()
numTokens := utilities.UserInputInteger("How many tokens do you wish to use? You have %v ", len(instances))
quit := make(chan bool)
var allQueries []string
var chars string
rawChars := " !\"#$%&'()*+,-./0123456789:;<=>?@[]^_`abcdefghijklmnopqrstuvwxyz{|}~" + cfg.ScraperSettings.ExtendedChars
for i := 0; i < len(rawChars); i++ {
if !strings.Contains(rawChars[0:i], string(rawChars[i])) {
chars += string(rawChars[i])
}
}
queriesLeft := make(chan string)
for i := 0; i < len(chars); i++ {
go func(i int) {
queriesLeft <- string(chars[i])
}(i)
}
if numTokens > len(instances) {
utilities.LogWarn("You only have %v tokens in your tokens.txt Using the maximum number of tokens possible", len(instances))
} else if numTokens <= 0 {
utilities.LogErr("You must atleast use 1 token")
utilities.ExitSafely()
} else if numTokens <= len(instances) {
utilities.LogInfo("You have %v tokens in your tokens.txt Using %v tokens", len(instances), numTokens)
instances = instances[:numTokens]
} else {
utilities.LogErr("Invalid input")
}
serverid := utilities.UserInput("Enter the server ID: ")
var tokenFile, botsFile, avatarFile, nameFile, path, rolePath, scrapedFile, userDataFile string
if cfg.OtherSettings.Logs {
path = fmt.Sprintf(`logs/offline_scraper/DMDGO-OS-%s-%s-%s`, serverid, time.Now().Format(`2006-01-02 15-04-05`), utilities.RandStringBytes(5))
rolePath = fmt.Sprintf(`%s/roles`, path)
err := os.MkdirAll(path, 0755)
if err != nil && !os.IsExist(err) {
utilities.LogErr("Error creating logs directory: %s", err)
utilities.ExitSafely()
}
err = os.MkdirAll(rolePath, 0755)
if err != nil && !os.IsExist(err) {
utilities.LogErr("Error creating roles directory: %s", err)
utilities.ExitSafely()
}
tokenFileX, err := os.Create(fmt.Sprintf(`%s/token.txt`, path))
if err != nil {
utilities.LogErr("Error creating token file: %s", err)
utilities.ExitSafely()
}
tokenFileX.Close()
botsFileX, err := os.Create(fmt.Sprintf(`%s/bots.txt`, path))
if err != nil {
utilities.LogErr("Error creating bots file: %s", err)
utilities.ExitSafely()
}
botsFileX.Close()
AvatarFileX, err := os.Create(fmt.Sprintf(`%s/avatars.txt`, path))
if err != nil {
utilities.LogErr("Error creating avatars file: %s", err)
utilities.ExitSafely()
}
AvatarFileX.Close()
NameFileX, err := os.Create(fmt.Sprintf(`%s/names.txt`, path))
if err != nil {
utilities.LogErr("Error creating names file: %s", err)
utilities.ExitSafely()
}
NameFileX.Close()
ScrapedFileX, err := os.Create(fmt.Sprintf(`%s/scraped.txt`, path))
if err != nil {
utilities.LogErr("Error creating scraped file: %s", err)
utilities.ExitSafely()
}
defer ScrapedFileX.Close()
UserDataFileX, err := os.Create(fmt.Sprintf(`%s/userdata.txt`, path))
if err != nil {
utilities.LogErr("Error creating userdata file: %s", err)
utilities.ExitSafely()
}
defer UserDataFileX.Close()
tokenFile, botsFile, avatarFile, nameFile, scrapedFile, userDataFile = tokenFileX.Name(), botsFileX.Name(), AvatarFileX.Name(), NameFileX.Name(), ScrapedFileX.Name(), UserDataFileX.Name()
for i := 0; i < len(instances); i++ {
instances[i].WriteInstanceToFile(tokenFile)
}
}
utilities.LogInfo("Press ENTER to START and STOP scraping")
bufio.NewReader(os.Stdin).ReadBytes('\n')
var namesScraped []string
var avatarsScraped []string
for i := 0; i < len(instances); i++ {
go func(i int) {
instances[i].ScrapeCount = 0
for {
if instances[i].ScrapeCount%5 == 0 || instances[i].LastCount%100 == 0 {
if instances[i].Ws != nil {
instances[i].Ws.Close()
}
time.Sleep(2 * time.Second)
err := instances[i].StartWS()
if err != nil {
fmt.Println(err)
continue
}
time.Sleep(2 * time.Second)
}
instances[i].ScrapeCount++
select {
case <-quit:
if instances[i].Ws != nil {
instances[i].Ws.Close()
}
return
default:
query := <-queriesLeft
allQueries = append(allQueries, query)
if instances[i].Ws == nil {
continue
}
if instances[i].Ws.Conn == nil {
continue
}
err := instance.ScrapeOffline(instances[i].Ws, serverid, query)
if err != nil {
utilities.LogErr("%v Error while scraping: %v", instances[i].CensorToken(), err)
go func() {
queriesLeft <- query
}()
continue
}
memInfo := <-instances[i].Ws.OfflineScrape
queriesCompleted = append(queriesCompleted, query)
var MemberInfo instance.Event
err = json.Unmarshal(memInfo, &MemberInfo)
if err != nil {
utilities.LogErr("Error while unmarshalling: %v", err)
queriesLeft <- query
continue
}
if len(MemberInfo.Data.Members) == 0 {
instances[i].LastCount = -1
continue
}
instances[i].LastCount = len(MemberInfo.Data.Members)
for _, member := range MemberInfo.Data.Members {
if !utilities.Contains(scraped, member.User.ID) {
scraped = append(scraped, member.User.ID)
}
}
utilities.LogSuccess("Token %v Query %v Scraped %v [+%v]", instances[i].CensorToken(), query, len(scraped), len(MemberInfo.Data.Members))
for i := 0; i < len(MemberInfo.Data.Members); i++ {
id := MemberInfo.Data.Members[i].User.ID
err := utilities.WriteLines("memberids.txt", id)
if err != nil {
utilities.LogErr("Error while writing to file: %v", err)
continue
}
if cfg.OtherSettings.Logs {
utilities.WriteLinesPath(scrapedFile, id)
if MemberInfo.Data.Members[i].User.Bot {
utilities.WriteLinesPath(botsFile, fmt.Sprintf("%v %v %v", id, MemberInfo.Data.Members[i].User.Username, MemberInfo.Data.Members[i].User.Discriminator))
}
if MemberInfo.Data.Members[i].User.Avatar != "" {
utilities.WriteLinesPath(avatarFile, fmt.Sprintf("%v:%v", id, MemberInfo.Data.Members[i].User.Avatar))
}
if MemberInfo.Data.Members[i].User.Username != "" {
utilities.WriteLinesPath(nameFile, fmt.Sprintf("%v", MemberInfo.Data.Members[i].User.Username))
}
for x := 0; x < len(MemberInfo.Data.Members[i].Roles); x++ {
utilities.WriteRoleFile(id, rolePath, MemberInfo.Data.Members[i].Roles[x])
}
if MemberInfo.Data.Members[i].User.Username != "" && MemberInfo.Data.Members[i].User.Discriminator != "" {
utilities.WriteLinesPath(userDataFile, fmt.Sprintf("%v#%v", MemberInfo.Data.Members[i].User.Username, MemberInfo.Data.Members[i].User.Discriminator))
}
}
if cfg.ScraperSettings.ScrapeUsernames {
nom := MemberInfo.Data.Members[i].User.Username
if !utilities.Contains(namesScraped, nom) {
err := utilities.WriteLines("names.txt", nom)
if err != nil {
utilities.LogErr("Error while writing to file: %v", err)
continue
}
}
}
if cfg.ScraperSettings.ScrapeAvatars {
av := MemberInfo.Data.Members[i].User.Avatar
if !utilities.Contains(avatarsScraped, av) {
err := utilities.ProcessAvatar(av, id)
if err != nil {
utilities.LogErr("Error while processing avatar: %v", err)
continue
}
}
}
}
if len(MemberInfo.Data.Members) < 100 {
time.Sleep(time.Duration(cfg.ScraperSettings.SleepSc) * time.Millisecond)
continue
}
lastName := MemberInfo.Data.Members[len(MemberInfo.Data.Members)-1].User.Username
nextQueries := instance.FindNextQueries(query, lastName, queriesCompleted, chars)
for i := 0; i < len(nextQueries); i++ {
go func(i int) {
queriesLeft <- nextQueries[i]
}(i)
}
}
}
}(i)
}
bufio.NewReader(os.Stdin).ReadBytes('\n')
utilities.LogInfo("Stopping All instances")
title <- true
for i := 0; i < len(instances); i++ {
go func() {
quit <- true
}()
}
utilities.LogInfo("Scraping Complete. %v members scraped.", len(scraped))
choice := utilities.UserInput("Do you wish to write to file again? (y/n) [This will remove pre-existing IDs from memberids.txt]")
if choice == "y" || choice == "Y" {
clean := utilities.RemoveDuplicateStr(scraped)
err := utilities.TruncateLines("memberids.txt", clean)
if err != nil {
utilities.LogErr("Error while truncating file: %v", err)
}
}
}
}
